Fix stuff

- cleanup cmd.Dump, with deprication notice
- don't prefic cmd-commands

changelog/unreleased/restore-backup.yml

@@ -0,0 +1,3 @@
+title: Implement proper `gitea restore` & `gitea backup`. Deprecating `gitea dump`
+kind: feature
+pull_request: 1637

cmd/backup.go

@@ -0,0 +1,197 @@
+// Copyright 2017 the Gitea Authors. All rights reserved.
+// Copyright 2017 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"runtime/debug"
+	"time"
+
+	"github.com/Unknwon/cae/zip"
+	"github.com/Unknwon/com"
+	"github.com/urfave/cli"
+	"gopkg.in/ini.v1"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// Backup files and database
+var Backup = cli.Command{
+	Name:  "backup",
+	Usage: "Backup files and database",
+	Description: `Backup dumps and compresses all related files and database into zip file,
+   which can be used for migrating Gitea to another server. The output format is meant to be
+   portable among all supported database engines.`,
+	Action: runBackup,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "config, c",
+			Value: "custom/conf/app.ini",
+			Usage: "Custom configuration `FILE` path",
+		},
+		cli.StringFlag{
+			Name:  "tempdir, t",
+			Value: os.TempDir(),
+			Usage: "Temporary directory `PATH`",
+		},
+		cli.StringFlag{
+			Name:  "target",
+			Value: "./",
+			Usage: "Target directory `PATH` to save backup archive",
+		},
+		cli.BoolFlag{
+			Name:  "verbose, v",
+			Usage: "Show process details",
+		},
+		cli.BoolTFlag{
+			Name:  "db",
+			Usage: "Backup the database (default: true)",
+		},
+		cli.BoolTFlag{
+			Name:  "repos",
+			Usage: "Backup repositories (default: true)",
+		},
+		cli.BoolTFlag{
+			Name:  "data",
+			Usage: "Backup attachments and avatars (default: true)",
+		},
+		cli.BoolTFlag{
+			Name:  "custom",
+			Usage: "Backup custom files (default: true)",
+		},
+	},
+}
+
+const (
+	archiveRootDir = "gitea-backup"
+	backupVersion  = 1
+)
+
+func runBackup(c *cli.Context) error {
+	zip.Verbose = c.Bool("verbose")
+	if c.IsSet("config") {
+		setting.CustomConf = c.String("config")
+	}
+	setting.NewContext()
+	models.LoadConfigs()
+	if err := models.SetEngine(); err != nil {
+		return err
+	}
+
+	// Setup temp-dir
+	tmpDir := c.String("tempdir")
+	if !com.IsExist(tmpDir) {
+		log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir)
+	}
+	rootDir, err := ioutil.TempDir(tmpDir, "gitea-backup-")
+	if err != nil {
+		log.Fatal(0, "Fail to create backup root directory '%s': %v", rootDir, err)
+	}
+	defer func(rootDir string) {
+		os.RemoveAll(rootDir)
+	}(rootDir)
+	log.Info("Backup root directory: %s", rootDir)
+
+	// Metadata
+	metaFile := path.Join(rootDir, "metadata.ini")
+	metadata := ini.Empty()
+	metadata.Section("").Key("VERSION").SetValue(fmt.Sprintf("%d", backupVersion))
+	metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String())
+	metadata.Section("").Key("GITEA_VERSION").SetValue(setting.AppVer)
+	if err = metadata.SaveTo(metaFile); err != nil {
+		log.Fatal(0, "Fail to save metadata '%s': %v", metaFile, err)
+	}
+
+	// Create ZIP-file
+	archiveName := path.Join(c.String("target"), fmt.Sprintf("gitea-backup-%d.zip", time.Now().Unix()))
+	log.Info("Packing backup files to: %s", archiveName)
+
+	z, err := zip.Create(archiveName)
+	if err != nil {
+		log.Fatal(0, "Fail to create backup archive '%s': %v", archiveName, err)
+	}
+	defer func(archiveName string) {
+		if r := recover(); r != nil {
+			var ok bool
+			err, ok = r.(error)
+			if !ok {
+				err = fmt.Errorf("pkg: %v", r)
+			}
+			debug.PrintStack()
+			log.Info("Removing partial backup-file %s\n", archiveName)
+			os.Remove(archiveName)
+			log.Fatal(9, "%v\n", err)
+		}
+	}(archiveName)
+
+	// Add metadata-file
+	if err = z.AddFile(archiveRootDir+"/metadata.ini", metaFile); err != nil {
+		log.Fatal(0, "Fail to include 'metadata.ini': %v", err)
+	}
+
+	// Database
+	if c.Bool("db") {
+		log.Info("Backing up database")
+		dbDir := path.Join(rootDir, "db")
+		if err = models.DumpDatabase(dbDir); err != nil {
+			log.Fatal(0, "Fail to dump database: %v", err)
+		}
+		if err = z.AddDir(archiveRootDir+"/db", dbDir); err != nil {
+			log.Fatal(0, "Fail to include 'db': %v", err)
+		}
+	}
+
+	// Custom files
+	if c.Bool("custom") {
+		log.Info("Backing up custom files")
+		if err = z.AddDir(archiveRootDir+"/custom", setting.CustomPath); err != nil {
+			log.Fatal(0, "Fail to include 'custom': %v", err)
+		}
+	}
+
+	// Data files
+	if c.Bool("data") {
+		log.Info("Backing up attachments and avatars")
+		for _, dir := range []string{"attachments", "avatars"} {
+			dirPath := path.Join(setting.AppDataPath, dir)
+			if !com.IsDir(dirPath) {
+				continue
+			}
+
+			if err = z.AddDir(path.Join(archiveRootDir+"/data", dir), dirPath); err != nil {
+				log.Fatal(0, "Fail to include 'data': %v", err)
+			}
+		}
+	}
+
+	// Repositories
+	if c.Bool("repos") {
+		log.Info("Backing up repositories")
+		reposDump := path.Join(rootDir, "repositories.zip")
+		log.Info("Dumping repositories in '%s'", setting.RepoRootPath)
+		if err = zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
+			log.Fatal(0, "Fail to dump repositories: %v", err)
+		}
+		log.Info("Repositories dumped to: %s", reposDump)
+
+		if err = z.AddFile(archiveRootDir+"/repositories.zip", reposDump); err != nil {
+			log.Fatal(0, "Fail to include 'repositories.zip': %v", err)
+		}
+	}
+
+	if err = z.Close(); err != nil {
+		log.Fatal(0, "Fail to save backup archive '%s': %v", archiveName, err)
+	}
+
+	os.RemoveAll(rootDir)
+	log.Info("Backup succeed! Archive is located at: %s", archiveName)
+	return nil
+}

cmd/dump.go

@@ -8,13 +8,13 @@ package cmd
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
 	"time"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/Unknwon/cae/zip"
@@ -22,10 +22,10 @@ import (
 	"github.com/urfave/cli"
 )
 
-// CmdDump represents the available dump sub-command.
-var CmdDump = cli.Command{
+// Dump represents the available dump sub-command.
+var Dump = cli.Command{
 	Name:  "dump",
-	Usage: "Dump Gitea files and database",
+	Usage: "DEPRICATED! Dump Gitea files and database",
 	Description: `Dump compresses all related files and database into zip file.
 It can be used for backup and capture Gitea server image to send to maintainer`,
 	Action: runDump,
@@ -59,65 +59,66 @@ func runDump(ctx *cli.Context) error {
 	setting.NewServices() // cannot access session settings otherwise
 	models.LoadConfigs()
 
-	err := models.SetEngine()
-	if err != nil {
+	log.Info("")
+
+	if err := models.SetEngine(); err != nil {
 		return err
 	}
 
 	tmpDir := ctx.String("tempdir")
 	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
-		log.Fatalf("Path does not exist: %s", tmpDir)
+		log.Fatal(4, "Path does not exist: %s", tmpDir)
 	}
 	TmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-")
 	if err != nil {
-		log.Fatalf("Failed to create tmp work directory: %v", err)
+		log.Fatal(4, "Failed to create tmp work directory: %v", err)
 	}
-	log.Printf("Creating tmp work dir: %s", TmpWorkDir)
+	log.Info("Creating tmp work dir: %s", TmpWorkDir)
 
 	reposDump := path.Join(TmpWorkDir, "gitea-repo.zip")
 	dbDump := path.Join(TmpWorkDir, "gitea-db.sql")
 
-	log.Printf("Dumping local repositories...%s", setting.RepoRootPath)
+	log.Info("Dumping local repositories...%s", setting.RepoRootPath)
 	zip.Verbose = ctx.Bool("verbose")
-	if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
-		log.Fatalf("Failed to dump local repositories: %v", err)
+	if err = zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
+		log.Fatal(4, "Failed to dump local repositories: %v", err)
 	}
 
 	targetDBType := ctx.String("database")
 	if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type {
-		log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType)
+		log.Info("Dumping database %s => %s...", models.DbCfg.Type, targetDBType)
 	} else {
-		log.Printf("Dumping database...")
+		log.Info("Dumping database...")
 	}
 
-	if err := models.DumpDatabase(dbDump, targetDBType); err != nil {
-		log.Fatalf("Failed to dump database: %v", err)
+	if err = models.DumpDatabaseOld(dbDump, targetDBType); err != nil {
+		log.Fatal(4, "Failed to dump database: %v", err)
 	}
 
 	fileName := fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix())
-	log.Printf("Packing dump files...")
+	log.Info("Packing dump files...")
 	z, err := zip.Create(fileName)
 	if err != nil {
-		log.Fatalf("Failed to create %s: %v", fileName, err)
+		log.Fatal(4, "Failed to create %s: %v", fileName, err)
 	}
 
-	if err := z.AddFile("gitea-repo.zip", reposDump); err != nil {
-		log.Fatalf("Failed to include gitea-repo.zip: %v", err)
+	if err = z.AddFile("gitea-repo.zip", reposDump); err != nil {
+		log.Fatal(4, "Failed to include gitea-repo.zip: %v", err)
 	}
-	if err := z.AddFile("gitea-db.sql", dbDump); err != nil {
-		log.Fatalf("Failed to include gitea-db.sql: %v", err)
+	if err = z.AddFile("gitea-db.sql", dbDump); err != nil {
+		log.Fatal(4, "Failed to include gitea-db.sql: %v", err)
 	}
 	customDir, err := os.Stat(setting.CustomPath)
 	if err == nil && customDir.IsDir() {
-		if err := z.AddDir("custom", setting.CustomPath); err != nil {
-			log.Fatalf("Failed to include custom: %v", err)
+		if err = z.AddDir("custom", setting.CustomPath); err != nil {
+			log.Fatal(4, "Failed to include custom: %v", err)
 		}
 	} else {
-		log.Printf("Custom dir %s doesn't exist, skipped", setting.CustomPath)
+		log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
 	}
 
 	if com.IsExist(setting.AppDataPath) {
-		log.Printf("Packing data directory...%s", setting.AppDataPath)
+		log.Info("Packing data directory...%s", setting.AppDataPath)
 
 		var sessionAbsPath string
 		if setting.SessionConfig.Provider == "file" {
@@ -126,30 +127,30 @@ func runDump(ctx *cli.Context) error {
 			}
 			sessionAbsPath, _ = filepath.Abs(setting.SessionConfig.ProviderConfig)
 		}
-		if err := zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil {
-			log.Fatalf("Failed to include data directory: %v", err)
+		if err = zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil {
+			log.Fatal(4, "Failed to include data directory: %v", err)
 		}
 	}
 
-	if err := z.AddDir("log", setting.LogRootPath); err != nil {
-		log.Fatalf("Failed to include log: %v", err)
+	if err = z.AddDir("log", setting.LogRootPath); err != nil {
+		log.Fatal(4, "Failed to include log: %v", err)
 	}
 
 	if err = z.Close(); err != nil {
 		_ = os.Remove(fileName)
-		log.Fatalf("Failed to save %s: %v", fileName, err)
+		log.Fatal(4, "Failed to save %s: %v", fileName, err)
 	}
 
 	if err := os.Chmod(fileName, 0600); err != nil {
-		log.Printf("Can't change file access permissions mask to 0600: %v", err)
+		log.Info("Can't change file access permissions mask to 0600: %v", err)
 	}
 
-	log.Printf("Removing tmp work dir: %s", TmpWorkDir)
+	log.Info("Removing tmp work dir: %s", TmpWorkDir)
 
 	if err := os.RemoveAll(TmpWorkDir); err != nil {
-		log.Fatalf("Failed to remove %s: %v", TmpWorkDir, err)
+		log.Fatal(4, "Failed to remove %s: %v", TmpWorkDir, err)
 	}
-	log.Printf("Finish dumping in file %s", fileName)
+	log.Info("Finish dumping in file %s", fileName)
 
 	return nil
 }

cmd/restore.go

@@ -0,0 +1,166 @@
+// Copyright 2017 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"os"
+	"path"
+
+	"github.com/Unknwon/cae/zip"
+	"github.com/Unknwon/com"
+	"github.com/mcuadros/go-version"
+	"github.com/urfave/cli"
+	"gopkg.in/ini.v1"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// Restore a backup
+var Restore = cli.Command{
+	Name:  "restore",
+	Usage: "Restore files and database from backup",
+	Description: `Restore imports all related files and database from a backup archive.
+The backup version must lower or equal to current Gitea version. You can also import
+backup from other database engines, which is useful for database migrating.
+
+If corresponding files or database tables are not presented in the archive, they will
+be skipped and remian unchanged.`,
+	Action: runRestore,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "config, c",
+			Value: "custom/conf/app.ini",
+			Usage: "Custom configuration file path",
+		},
+		cli.StringFlag{
+			Name:  "tempdir, t",
+			Value: os.TempDir(),
+			Usage: "Temporary directory path",
+		},
+		cli.StringFlag{
+			Name:  "from",
+			Value: "",
+			Usage: "Path to backup archive",
+		},
+		cli.BoolFlag{
+			Name:  "verbose, v",
+			Usage: "Show process details",
+		},
+		cli.BoolTFlag{
+			Name:  "repos",
+			Usage: "Restore repositories (default: true)",
+		},
+		cli.BoolTFlag{
+			Name:  "data",
+			Usage: "Restore attachments and avatars (default: true)",
+		},
+		cli.BoolTFlag{
+			Name:  "custom",
+			Usage: "Restore custom files (default: true)",
+		},
+		cli.BoolTFlag{
+			Name:  "db",
+			Usage: "Restore database (default: true)",
+		},
+	},
+}
+
+func runRestore(c *cli.Context) error {
+	zip.Verbose = c.Bool("verbose")
+
+	tmpDir := c.String("tempdir")
+	if !com.IsExist(tmpDir) {
+		log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir)
+	}
+
+	log.Info("Restore backup from: %s", c.String("from"))
+	if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil {
+		log.Fatal(0, "Fail to extract backup archive: %v", err)
+	}
+	archivePath := path.Join(tmpDir, archiveRootDir)
+
+	// Check backup version
+	metaFile := path.Join(archivePath, "metadata.ini")
+	if !com.IsExist(metaFile) {
+		log.Fatal(0, "File 'metadata.ini' is missing")
+	}
+	metadata, err := ini.Load(metaFile)
+	if err != nil {
+		log.Fatal(0, "Fail to load metadata '%s': %v", metaFile, err)
+	}
+	ver := metadata.Section("").Key("VERSION").MustInt(10000000)
+	if ver != backupVersion {
+		log.Fatal(0, "Current Backup version does not match the version in the backup: %d != %d", ver, backupVersion)
+	}
+	backupVersion := metadata.Section("").Key("GITEA_VERSION").MustString("999.0")
+	if version.Compare(setting.AppVer, backupVersion, "<") {
+		log.Fatal(0, "Current Gitea version is lower than backup version: %s < %s", setting.AppVer, backupVersion)
+	}
+
+	// If config file is not present in backup, user must set this file via flag.
+	// Otherwise, it's optional to set config file flag.
+	configFile := path.Join(archivePath, "custom/conf/app.ini")
+	if c.IsSet("config") {
+		setting.CustomConf = c.String("config")
+	} else if !com.IsExist(configFile) {
+		log.Fatal(0, "'--config' is not specified and custom config file is not found in backup")
+	} else {
+		setting.CustomConf = configFile
+	}
+	setting.NewContext()
+	models.LoadConfigs()
+	models.SetEngine()
+
+	// Database
+	if c.Bool("db") {
+		dbDir := path.Join(archivePath, "db")
+		if err = models.ImportDatabase(dbDir); err != nil {
+			log.Fatal(0, "Fail to import database: %v", err)
+		}
+	}
+
+	// Custom files
+	if c.Bool("custom") {
+		if com.IsExist(setting.CustomPath) {
+			if err = os.Rename(setting.CustomPath, setting.CustomPath+".bak"); err != nil {
+				log.Fatal(0, "Fail to backup current 'custom': %v", err)
+			}
+		}
+		if err = os.Rename(path.Join(archivePath, "custom"), setting.CustomPath); err != nil {
+			log.Fatal(0, "Fail to import 'custom': %v", err)
+		}
+	}
+
+	// Data files
+	if c.Bool("data") {
+		for _, dir := range []string{"attachments", "avatars"} {
+			dirPath := path.Join(setting.AppDataPath, dir)
+			if com.IsExist(dirPath) {
+				if err = os.Rename(dirPath, dirPath+".bak"); err != nil {
+					log.Fatal(0, "Fail to backup current 'data': %v", err)
+				}
+			}
+			if err = os.Rename(path.Join(archivePath, "data", dir), dirPath); err != nil {
+				log.Fatal(0, "Fail to import 'data': %v", err)
+			}
+		}
+	}
+
+	// Repositories
+	if c.Bool("repos") {
+		reposPath := path.Join(archivePath, "repositories.zip")
+		if !c.Bool("exclude-repos") && !c.Bool("database-only") && com.IsExist(reposPath) {
+			if err := zip.ExtractTo(reposPath, path.Dir(setting.RepoRootPath)); err != nil {
+				log.Fatal(0, "Fail to extract 'repositories.zip': %v", err)
+			}
+		}
+	}
+
+	os.RemoveAll(path.Join(tmpDir, archiveRootDir))
+	log.Info("Restore succeed!")
+	return nil
+}

main.go

@@ -36,9 +36,11 @@ func main() {
 		cmd.CmdWeb,
 		cmd.CmdServ,
 		cmd.CmdHook,
-		cmd.CmdDump,
 		cmd.CmdCert,
 		cmd.CmdAdmin,
+		cmd.Backup,
+		cmd.Restore,
+		cmd.Dump,
 	}
 	app.Flags = append(app.Flags, []cli.Flag{}...)
 	err := app.Run(os.Args)

models/models.go

@@ -5,7 +5,9 @@
 package models
 
 import (
+	"bufio"
 	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
@@ -13,20 +15,21 @@ import (
 	"path"
 	"strings"
 
-	// Needed for the MySQL driver
-	_ "github.com/go-sql-driver/mysql"
+	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/Unknwon/com"
 	"github.com/go-xorm/core"
 	"github.com/go-xorm/xorm"
+	// Needed for the MySQL driver
+	_ "github.com/go-sql-driver/mysql"
 
 	// Needed for the Postgresql driver
 	_ "github.com/lib/pq"
 
 	// Needed for the MSSSQL driver
 	_ "github.com/denisenkom/go-mssqldb"
-
-	"code.gitea.io/gitea/models/migrations"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
 )
 
 // Engine represents a xorm engine or session.
@@ -327,14 +330,104 @@ func Ping() error {
 	return x.Ping()
 }
 
-// DumpDatabase dumps all data from database according the special database SQL syntax to file system.
-func DumpDatabase(filePath string, dbType string) error {
+// Version describes the version table. Should have only one row with id==1
+type Version struct {
+	ID      int64 `xorm:"pk autoincr"`
+	Version int64
+}
+
+// DumpDatabaseOld dumps all data from database according the special database SQL syntax to file system.
+// NOTE: DEPRICATED in favour of DumpDatabase
+func DumpDatabaseOld(filePath string, dbType string) error {
 	var tbs []*core.Table
 	for _, t := range tables {
 		tbs = append(tbs, x.TableInfo(t).Table)
+		if len(dbType) > 0 {
+			return x.DumpTablesToFile(tbs, filePath, core.DbType(dbType))
+		}
 	}
-	if len(dbType) > 0 {
-		return x.DumpTablesToFile(tbs, filePath, core.DbType(dbType))
-	}
-	return x.DumpTablesToFile(tbs, filePath)
+	return nil
+}
+
+// DumpDatabase dumps all data from database to file system in JSON format.
+func DumpDatabase(dirPath string) (err error) {
+	os.MkdirAll(dirPath, os.ModePerm)
+	// Purposely create a local variable to not modify global variable
+	internalTables := append(tables, new(Version))
+	for _, table := range internalTables {
+		tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.")
+		tableFile := path.Join(dirPath, tableName+".json")
+		f, err := os.Create(tableFile)
+		if err != nil {
+			return fmt.Errorf("fail to create JSON file: %v", err)
+		}
+
+		if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) {
+			enc := json.NewEncoder(f)
+			return enc.Encode(bean)
+		}); err != nil {
+			f.Close()
+			return fmt.Errorf("fail to dump table '%s': %v", tableName, err)
+		}
+		f.Close()
+	}
+	return nil
+}
+
+// ImportDatabase imports data from backup archive.
+func ImportDatabase(dirPath string) (err error) {
+	// Purposely create a local variable to not modify global variable
+	internalTables := append(tables, new(Version))
+	for _, table := range internalTables {
+		tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.")
+		tableFile := path.Join(dirPath, tableName+".json")
+		if !com.IsExist(tableFile) {
+			continue
+		}
+
+		if err = x.DropTables(table); err != nil {
+			return fmt.Errorf("fail to drop table '%s': %v", tableName, err)
+		} else if err = x.Sync2(table); err != nil {
+			return fmt.Errorf("fail to sync table '%s': %v", tableName, err)
+		}
+
+		f, err := os.Open(tableFile)
+		if err != nil {
+			return fmt.Errorf("fail to open JSON file: %v", err)
+		}
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			switch bean := table.(type) {
+			case *LoginSource:
+				meta := make(map[string]interface{})
+				if err = json.Unmarshal(scanner.Bytes(), &meta); err != nil {
+					return fmt.Errorf("fail to unmarshal to map: %v", err)
+				}
+
+				tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64())
+				switch tp {
+				case LoginLDAP, LoginDLDAP:
+					bean.Cfg = new(LDAPConfig)
+				case LoginSMTP:
+					bean.Cfg = new(SMTPConfig)
+				case LoginPAM:
+					bean.Cfg = new(PAMConfig)
+				case LoginOAuth2:
+					bean.Cfg = new(OAuth2Config)
+				default:
+					return fmt.Errorf("unrecognized login source type:: %v", tp)
+				}
+				table = bean
+			}
+
+			if err = json.Unmarshal(scanner.Bytes(), table); err != nil {
+				return fmt.Errorf("fail to unmarshal to struct: %v", err)
+			}
+
+			if _, err = x.Insert(table); err != nil {
+				return fmt.Errorf("fail to insert strcut: %v", err)
+			}
+		}
+	}
+	return nil
 }