Refactor path & config system (go-gitea#25330)

# The problem

There were many "path tricks":

* By default, Gitea uses its program directory as its work path
* Gitea tries to use the "work path" to guess its "custom path" and
"custom conf (app.ini)"
* Users might want to use other directories as work path
* The non-default work path should be passed to Gitea by GITEA_WORK_DIR
or "--work-path"
* But some Gitea processes are started without these values
    * The "serv" process started by OpenSSH server
    * The CLI sub-commands started by site admin
* The paths are guessed by SetCustomPathAndConf again and again
* The default values of "work path / custom path / custom conf" can be
changed when compiling

# The solution

* Use `InitWorkPathAndCommonConfig` to handle these path tricks, and use
test code to cover its behaviors.
* When Gitea's web server runs, write the WORK_PATH to "app.ini", this
value must be the most correct one, because if this value is not right,
users would find that the web UI doesn't work and then they should be
able to fix it.
* Then all other sub-commands can use the WORK_PATH in app.ini to
initialize their paths.
* By the way, when Gitea starts for git protocol, it shouldn't output
any log, otherwise the git protocol gets broken and client blocks
forever.

The "work path" priority is: WORK_PATH in app.ini > cmd arg --work-path
> env var GITEA_WORK_DIR > builtin default

The "app.ini" searching order is: cmd arg --config > cmd arg "work path
/ custom path" > env var "work path / custom path" > builtin default

## ⚠️ BREAKING

If your instance's "work path / custom path / custom conf" doesn't meet
the requirements (eg: work path must be absolute), Gitea will report a
fatal error and exit. You need to set these values according to the
error log.

----

Close go-gitea#24818
Close go-gitea#24222
Close go-gitea#21606
Close go-gitea#21498
Close go-gitea#25107
Close go-gitea#24981
Maybe close go-gitea#24503

Replace go-gitea#23301
Replace go-gitea#22754

And maybe more
# Conflicts:
#	cmd/web.go

.gitignore

@@ -53,8 +53,6 @@ cpu.out
 /bin
 /dist
 /custom/*
-!/custom/conf
-/custom/conf/*
 !/custom/conf/app.example.ini
 /data
 /indexers

cmd/actions.go

@@ -42,7 +42,7 @@ func runGenerateActionsRunnerToken(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	scope := c.String("scope")
 

cmd/cmd.go

@@ -58,7 +58,7 @@ func confirm() (bool, error) {
 }
 
 func initDB(ctx context.Context) error {
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 	setting.InitSQLLoggersForCli(log.INFO)
 

cmd/doctor.go

@@ -91,7 +91,7 @@ func runRecreateTable(ctx *cli.Context) error {
 	golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info))
 
 	debug := ctx.Bool("debug")
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 
 	if debug {

cmd/dump.go

@@ -182,7 +182,7 @@ func runDump(ctx *cli.Context) error {
 		}
 		fileName += "." + outType
 	}
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	// make sure we are logging to the console no matter what the configuration tells us do to
 	// FIXME: don't use CfgProvider directly

cmd/embedded.go

@@ -99,11 +99,6 @@ type assetFile struct {
 func initEmbeddedExtractor(c *cli.Context) error {
 	setupConsoleLogger(log.ERROR, log.CanColorStderr, os.Stderr)
 
-	// Read configuration file
-	setting.Init(&setting.Options{
-		AllowEmpty: true,
-	})
-
 	patterns, err := compileCollectPatterns(c.Args())
 	if err != nil {
 		return err

cmd/mailer.go

@@ -16,7 +16,7 @@ func runSendMail(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	if err := argsSet(c, "title"); err != nil {
 		return err

cmd/restore_repo.go

@@ -51,7 +51,7 @@ func runRestoreRepository(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	var units []string
 	if s := c.String("units"); s != "" {
 		units = strings.Split(s, ",")

cmd/serv.go

@@ -61,7 +61,7 @@ func setup(ctx context.Context, debug bool) {
 	} else {
 		setupConsoleLogger(log.FATAL, false, os.Stderr)
 	}
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	if debug {
 		setting.RunMode = "dev"
 	}

cmd/web.go

@@ -101,6 +101,110 @@ func createPIDFile(pidPath string) {
 	}
 }
 
+func serveInstall(ctx *cli.Context) error {
+	log.Info("Gitea version: %s%s", setting.AppVer, setting.AppBuiltWith)
+	log.Info("App path: %s", setting.AppPath)
+	log.Info("Work path: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Config file: %s", setting.CustomConf)
+	log.Info("Prepare to run install page")
+
+	routers.InitWebInstallPage(graceful.GetManager().HammerContext())
+
+	// Flag for port number in case first time run conflict
+	if ctx.IsSet("port") {
+		if err := setPort(ctx.String("port")); err != nil {
+			return err
+		}
+	}
+	if ctx.IsSet("install-port") {
+		if err := setPort(ctx.String("install-port")); err != nil {
+			return err
+		}
+	}
+	c := install.Routes()
+	err := listen(c, false)
+	if err != nil {
+		log.Critical("Unable to open listener for installer. Is Gitea already running?")
+		graceful.GetManager().DoGracefulShutdown()
+	}
+	select {
+	case <-graceful.GetManager().IsShutdown():
+		<-graceful.GetManager().Done()
+		log.Info("PID: %d Gitea Web Finished", os.Getpid())
+		log.GetManager().Close()
+		return err
+	default:
+	}
+	return nil
+}
+
+func serveInstalled(ctx *cli.Context) error {
+	setting.InitCfgProvider(setting.CustomConf)
+	setting.LoadCommonSettings()
+	setting.MustInstalled()
+
+	log.Info("Gitea version: %s%s", setting.AppVer, setting.AppBuiltWith)
+	log.Info("App path: %s", setting.AppPath)
+	log.Info("Work path: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Config file: %s", setting.CustomConf)
+	log.Info("Run mode: %s", setting.RunMode)
+	log.Info("Prepare to run web server")
+
+	if setting.AppWorkPathMismatch {
+		log.Error("WORK_PATH from config %q doesn't match other paths from environment variables or command arguments. "+
+			"Only WORK_PATH in config should be set and used. Please remove the other outdated work paths from environment variables and command arguments", setting.CustomConf)
+	}
+
+	rootCfg := setting.CfgProvider
+	if rootCfg.Section("").Key("WORK_PATH").String() == "" {
+		saveCfg, err := rootCfg.PrepareSaving()
+		if err != nil {
+			log.Error("Unable to prepare saving WORK_PATH=%s to config %q: %v\nYou must set it manually, otherwise there might be bugs when accessing the git repositories.", setting.AppWorkPath, setting.CustomConf, err)
+		} else {
+			rootCfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath)
+			saveCfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath)
+			if err = saveCfg.Save(); err != nil {
+				log.Error("Unable to update WORK_PATH=%s to config %q: %v\nYou must set it manually, otherwise there might be bugs when accessing the git repositories.", setting.AppWorkPath, setting.CustomConf, err)
+			}
+		}
+	}
+
+	routers.InitWebInstalled(graceful.GetManager().HammerContext())
+
+	// We check that AppDataPath exists here (it should have been created during installation)
+	// We can't check it in `InitWebInstalled`, because some integration tests
+	// use cmd -> InitWebInstalled, but the AppDataPath doesn't exist during those tests.
+	if _, err := os.Stat(setting.AppDataPath); err != nil {
+		log.Fatal("Can not find APP_DATA_PATH %q", setting.AppDataPath)
+	}
+
+	// Override the provided port number within the configuration
+	if ctx.IsSet("port") {
+		if err := setPort(ctx.String("port")); err != nil {
+			return err
+		}
+	}
+
+	// Set up Chi routes
+	c := routers.NormalRoutes()
+	err := listen(c, true)
+	<-graceful.GetManager().Done()
+	log.Info("PID: %d Gitea Web Finished", os.Getpid())
+	log.GetManager().Close()
+	return err
+}
+
+func servePprof() {
+	http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
+	_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
+	// The pprof server is for debug purpose only, it shouldn't be exposed on public network. At the moment it's not worth to introduce a configurable option for it.
+	log.Info("Starting pprof server on localhost:6060")
+	log.Info("Stopped pprof server: %v", http.ListenAndServe("localhost:6060", nil))
+	finished()
+}
+
 func runWeb(ctx *cli.Context) error {
 	if ctx.Bool("verbose") {
 		setupConsoleLogger(log.TRACE, log.CanColorStdout, os.Stdout)
@@ -128,75 +232,19 @@ func runWeb(ctx *cli.Context) error {
 		createPIDFile(ctx.String("pid"))
 	}
 
-	// Perform pre-initialization
-	needsInstall := install.PreloadSettings(graceful.GetManager().HammerContext())
-	if needsInstall {
-		// Flag for port number in case first time run conflict
-		if ctx.IsSet("port") {
-			if err := setPort(ctx.String("port")); err != nil {
-				return err
-			}
-		}
-		if ctx.IsSet("install-port") {
-			if err := setPort(ctx.String("install-port")); err != nil {
-				return err
-			}
-		}
-		c := install.Routes()
-		err := listen(c, false)
-		if err != nil {
-			log.Critical("Unable to open listener for installer. Is Gitea already running?")
-			graceful.GetManager().DoGracefulShutdown()
-		}
-		select {
-		case <-graceful.GetManager().IsShutdown():
-			<-graceful.GetManager().Done()
-			log.Info("PID: %d Gitea Web Finished", os.Getpid())
-			log.GetManager().Close()
+	if !setting.InstallLock {
+		if err := serveInstall(ctx); err != nil {
 			return err
-		default:
 		}
 	} else {
 		NoInstallListener()
 	}
 
 	if setting.EnablePprof {
-		go func() {
-			http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
-			_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
-			// The pprof server is for debug purpose only, it shouldn't be exposed on public network. At the moment it's not worth to introduce a configurable option for it.
-			log.Info("Starting pprof server on localhost:6060")
-			log.Info("Stopped pprof server: %v", http.ListenAndServe("localhost:6060", nil))
-			finished()
-		}()
+		go servePprof()
 	}
 
-	log.Info("Global init")
-	// Perform global initialization
-	setting.Init(&setting.Options{})
-	routers.GlobalInitInstalled(graceful.GetManager().HammerContext())
-
-	// We check that AppDataPath exists here (it should have been created during installation)
-	// We can't check it in `GlobalInitInstalled`, because some integration tests
-	// use cmd -> GlobalInitInstalled, but the AppDataPath doesn't exist during those tests.
-	if _, err := os.Stat(setting.AppDataPath); err != nil {
-		log.Fatal("Can not find APP_DATA_PATH '%s'", setting.AppDataPath)
-	}
-
-	// Override the provided port number within the configuration
-	if ctx.IsSet("port") {
-		if err := setPort(ctx.String("port")); err != nil {
-			return err
-		}
-	}
-
-	// Set up Chi routes
-	c := routers.NormalRoutes(graceful.GetManager().HammerContext())
-	err := listen(c, true)
-	<-graceful.GetManager().Done()
-	log.Info("PID: %d Gitea Web Finished", os.Getpid())
-	log.GetManager().Close()
-	return err
+	return serveInstalled(ctx)
 }
 
 func setPort(port string) error {

contrib/environment-to-ini/environment-to-ini.go

@@ -81,21 +81,20 @@ func main() {
 		},
 	}
 	app.Action = runEnvironmentToIni
-	setting.SetCustomPathAndConf("", "", "")
-
 	err := app.Run(os.Args)
 	if err != nil {
 		log.Fatal("Failed to run app with %s: %v", os.Args, err)
 	}
 }
 
 func runEnvironmentToIni(c *cli.Context) error {
-	providedCustom := c.String("custom-path")
-	providedConf := c.String("config")
-	providedWorkPath := c.String("work-path")
-	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
+	setting.InitWorkPathAndCommonConfig(os.Getenv, setting.ArgWorkPathAndCustomConf{
+		WorkPath:   c.String("work-path"),
+		CustomPath: c.String("custom-path"),
+		CustomConf: c.String("config"),
+	})
 
-	cfg, err := setting.NewConfigProviderFromFile(&setting.Options{CustomConf: setting.CustomConf, AllowEmpty: true})
+	cfg, err := setting.NewConfigProviderFromFile(setting.CustomConf)
 	if err != nil {
 		log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
 	}