Use new method of obtaining login users

Author: mjarkk

README.md

@@ -63,12 +63,24 @@ Create a `env.json` file with the following content **(this file can also be obt
     },
     "alternative_servers": [
         // If you want to send CVs to multiple servers you can add additional servers here
+        // {"server_location": "http://localhost:4000","api_key_id": "aa","api_key": "bbb"}
     ],
-    "login_users": [
-        {"username": "scraping-site-username", "password": "scraping-site-password"}
+}
+```
+
+#### `2.1.` Mocking RTCV
+
+When developing a scraper you might want to mock RT-CV so you don't have to relay on another service.
+
+You can mock RT-CV by changing your env to this:
+
+```js
+{
+    "mock_mode": true,
+    "mock_users": [
+        // Here you place the login users for the site you are scraping that will be used as mock data
+        {"username": "scraping-site-username", "password": "scraping-site-password"},
     ],
-    // For production, set mock_mode to false
-    // "mock_mode": false
 }
 ```
 
@@ -99,7 +111,7 @@ Sends a cv to rtcv and remembers the reference number
 
 ### `$SCRAPER_ADDRESS/users`
 
-Returns the login users from the `env.json`
+Returns the login scraper users from RT-CV
 
 - Body: None
 - Resp: **true** / **false** the login users from env.json
@@ -127,6 +139,14 @@ Check if a reference number is in the cache
 - Body: The reference number
 - Resp: **true** / **false**
 
+## `env.json` is in another dir or has another name?
+
+You can change the credentials file location using this shell variable
+
+```sh
+export RTCV_SCRAPER_CLIENT_ENV_FILE=some/other/env/file.json
+```
+
 ## `env.json` as env variable?
 
 If you are using containers you might want to use the contents of your `env.json` as env variable instaid of binding it to the container as volume.

examples/deno/env.json

@@ -4,11 +4,5 @@
         "api_key_id": "aa",
         "api_key": "bbb"
     },
-    "alternative_servers": [],
-    "login_users": [
-        {
-            "username": "scraping-site-username",
-            "password": "scraping-site-password"
-        }
-    ]
+    "alternative_servers": []
 }

main.go

@@ -8,18 +8,30 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"sync"
 	"syscall"
 )
 
 // Env contains the structure of the .env file
 type Env struct {
 	PrimaryServer      EnvServer   `json:"primary_server"`
 	AlternativeServers []EnvServer `json:"alternative_servers"`
-	LoginUsers         []EnvUser   `json:"login_users"`
 	MockMode           bool        `json:"mock_mode"`
+	MockUsers          []EnvUser   `json:"mock_users"`
 }
 
 func (e *Env) validate() error {
+	if e.MockMode {
+		if len(e.MockUsers) == 0 {
+			fmt.Println("\"mock_users\" is empty in env.json, most scrapers require at least one user to login.")
+			fmt.Println("For documentation about mocking see https://github.com/script-development/rtcv_scraper_client")
+			if e.MockUsers == nil {
+				e.MockUsers = []EnvUser{}
+			}
+		}
+		return nil
+	}
+
 	err := e.PrimaryServer.validate()
 	if err != nil {
 		return fmt.Errorf("primary_server.%s", err.Error())
@@ -73,6 +85,11 @@ type EnvUser struct {
 
 func main() {
 	envFilename := "env.json"
+	alternativeFileName := os.Getenv("RTCV_SCRAPER_CLIENT_ENV_FILE")
+	if alternativeFileName != "" {
+		envFilename = alternativeFileName
+	}
+
 	envEnvName := "RTCV_SCRAPER_CLIENT_ENV"
 	envFile, err := ioutil.ReadFile(envFilename)
 	if err != nil {
@@ -97,16 +114,13 @@ func main() {
 	}
 
 	api := NewAPI()
-	useAddress := startWebserver(env, api)
 
 	credentials := []SetCredentialsArg{env.PrimaryServer.toCredArg(true)}
 	for _, server := range env.AlternativeServers {
 		credentials = append(credentials, server.toCredArg(false))
 	}
 
-	// Turn on mock mode by default
-	api.SetMockMode()
-
+	loginUsers := []EnvUser{}
 	if !env.MockMode {
 		err = api.SetCredentials(credentials)
 		if err != nil {
@@ -115,13 +129,17 @@ func main() {
 
 		fmt.Println("credentials set")
 		fmt.Println("testing connections..")
-		testServerConnections(api)
+		loginUsers = testServerConnections(api, credentials[0].APIKeyID)
 		fmt.Println("connected to RTCV")
 	} else {
+		api.SetMockMode()
+		loginUsers = env.MockUsers
 		fmt.Println("In mock mode")
-		fmt.Println("You can turn this off in by setting mock_mode to false in your env.json")
+		fmt.Println("You can turn this off in `env.json` by setting `mock_mode` to false")
 	}
 
+	useAddress := startWebserver(env, api, loginUsers)
+
 	fmt.Println("running scraper..")
 
 	if len(os.Args) <= 1 {
@@ -147,32 +165,53 @@ func main() {
 	}
 }
 
-func testServerConnections(api *API) {
+func testServerConnections(api *API, apiKeyID string) []EnvUser {
+	var wg sync.WaitGroup
+
 	for _, conn := range api.connections {
-		err := conn.Get("/api/v1/health", nil)
-		if err != nil {
-			log.Fatal(err)
-		}
+		wg.Add(1)
+		go func(conn serverConn) {
+			err := conn.Get("/api/v1/health", nil)
+			if err != nil {
+				log.Fatal(err)
+			}
 
-		apiKeyInfo := struct {
-			Roles []struct {
-				Role uint64 `json:"role"`
-			} `json:"roles"`
-		}{}
-		err = conn.Get("/api/v1/auth/keyinfo", &apiKeyInfo)
-		if err != nil {
-			log.Fatal(err)
-		}
+			apiKeyInfo := struct {
+				Roles []struct {
+					Role uint64 `json:"role"`
+				} `json:"roles"`
+			}{}
+			err = conn.Get("/api/v1/auth/keyinfo", &apiKeyInfo)
+			if err != nil {
+				log.Fatal(err)
+			}
 
-		hasScraperRole := false
-		for _, role := range apiKeyInfo.Roles {
-			if role.Role == 1 {
-				hasScraperRole = true
-				break
+			hasScraperRole := false
+			for _, role := range apiKeyInfo.Roles {
+				if role.Role == 1 {
+					hasScraperRole = true
+					break
+				}
 			}
-		}
-		if !hasScraperRole {
-			log.Fatal("provided key does not have scraper role (nr 1)")
-		}
+			if !hasScraperRole {
+				log.Fatal("provided key does not have scraper role (nr 1)")
+			}
+			wg.Done()
+		}(conn)
 	}
+
+	scraperUsers := struct {
+		Users []EnvUser `json:"users"`
+	}{}
+	err := api.connections[0].Get("/api/v1/scraperUsers/"+apiKeyID, &scraperUsers)
+
+	// Wait for the connections above to complete checking before we do this error check but do the request already so we don't have to wait for that
+	// If one of the connections has an error they will throw
+	wg.Wait()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return scraperUsers.Users
 }

webserver.go

@@ -11,8 +11,8 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-func startWebserver(env Env, api *API) string {
-	loginUsers, err := json.Marshal(env.LoginUsers)
+func startWebserver(env Env, api *API, loginUsers []EnvUser) string {
+	loginUsersJSON, err := json.Marshal(loginUsers)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -78,7 +78,7 @@ func startWebserver(env Env, api *API) string {
 
 			ctx.Response.AppendBodyString("true")
 		case "/users":
-			ctx.Response.AppendBody(loginUsers)
+			ctx.Response.AppendBody(loginUsersJSON)
 		case "/set_cached_reference", "/set_short_cached_reference":
 			refNr := string(ctx.Request.Body())
 			if refNr == "" {