Merge pull request #25 from DavidePrincipi/tier-id-calc-fix

Autoupdate policy implementation (beta6)

Author: DavidePrincipi

porthos/README.md

@@ -140,6 +140,41 @@ If `icat` field is not set, the `subscription.php` replies with 403 - forbidden.
 If `secret` field is not set, both `auth.php` and `subscription.php` reply with
 403 - forbidden, unless `$config['legacy_auth']` is enabled.
 
+## Porthos repository autoupdate policy
+
+When a client accesses the `autoupdate/` contents it is possible to
+differentiate  what it can see, on a client `tier_id` and `version` basis.
+
+The **default** policy is to return the repository state of the last monday, or
+the previous one, depending on the tier number. Custom policies can be
+configured by adding items to the `$config['autoupdate_policy']` array.
+
+Example 1:
+
+```php
+$config['autoupdate_policy'] = array(
+    '7.6.1810/0' => 'head',
+    '7.6.1810/*' => 'empty',
+);
+```
+
+This setting says that clients of tier 0 requesting version 7.6.1810 can see the
+`head` repository state. Clients of other tiers (identified by `*`) always see
+an `empty` repository.
+
+Example 2:
+
+```php
+$config['autoupdate_policy'] = array(
+    '7.6.1810/*' => 'default',
+    '7.6.1810/2' => 'fixed/d20191030',
+);
+```
+
+In this case clients of tier 2 see the repository state as it was on 2019-10-30.
+Other clients see the repository state according to the default policy (note
+that the corresponding line can be omitted because it already corresponds to the
+default policy).
 
 ## Repository management commands
 

porthos/root/srv/porthos/script/auth.php

@@ -58,14 +58,14 @@
 $valid_credentials = $valid_credentials && ($access['tier_id'] !== FALSE);
 
 if($access['tier_id'] < 0) {
-    $hash = 0;
+    $hash = $config['tier_seed'] ?: 0;
     foreach(str_split($_SERVER['PHP_AUTH_USER']) as $c) {
         $hash += ord($c);
     }
-    $hash = $hash % 256;
-    if($hash < 26) { // 10%
+    $hash = $hash % 10;
+    if($hash < 1) { // 10%
         $tier_id = 0;
-    } elseif($hash < 77) { // +20% = 30%
+    } elseif($hash < 3) { // +20% = 30%
         $tier_id = 1;
     } else { // +70% = 100%
         $tier_id = 2;
@@ -78,7 +78,7 @@
 if($is_tier_request && $valid_credentials) {
     // Seeking a snapshot is a time-consuming op. Ensure we have valid
     // credentials before running it!
-    $snapshot = lookup_snapshot($uri['full_path'], $tier_id, $config['week_size']);
+    $snapshot = lookup_snapshot($uri['full_path'], $uri['version'], $tier_id);
 } else {
     $snapshot = 'head';
 }
@@ -108,4 +108,8 @@
 }
 
 header('Cache-Control: private');
-return_file('/' . $snapshot . $uri['full_path']);
+if($snapshot == 'empty') {
+    return_file('/empty/repodata/' . basename($uri['full_path']));
+} else {
+    return_file('/' . $snapshot . $uri['full_path']);
+}

porthos/root/srv/porthos/script/config-porthos.php

@@ -56,7 +56,20 @@
 //     the PHP timezone for this application
 $config['timezone'] = 'UTC';
 
-// stop_autoupdate (array)
-//     forcibly stop autoupdate contents distribution for the listed versions.
-//     The /empty repository is served instead.
-$config['stop_autoupdate'] = array();
+// autoupdate_policy (array)
+//     Decide the content returned for a given version/tier_id combination.
+//     Valid keys are, for instance "7.6.1810/2" "6.10/0" "7.7.1908/*" ...
+//     Valid values are "head", "empty", "fixed/d20191104" (fixed snapshot name -
+//     if the snapshot dir does not exist, "empty" is assumed) and "default"
+//     (automatically selects the previous monday snapshot).
+$config['autoupdate_policy'] = array();
+
+// snapshots_dir (string)
+//     Filesystem directory path where snapshot directories are stored with
+//     trailing slash.
+$config['snapshots_dir'] = '/srv/porthos/webroot/';
+
+// tier_seed (int)
+//     The system_id value is reduced to the integer range 0-9 plus the seed.
+//     The resulting value is mapped to a tier_id
+$config['tier_seed'] = 0;

porthos/root/srv/porthos/script/lib.php

@@ -79,8 +79,49 @@ function get_snapshot_timestamp($snapshot_name) {
     return mktime(0, 0, 0, $parts['month'], $parts['day'], $parts['year']);
 }
 
-function lookup_snapshot($path, $tier_id = 0, $week_size = 5) {
-    $root_path = "/srv/porthos/webroot/";
+function lookup_policy($version, $tier_id) {
+    global $config;
+
+    $policy_map = $config['autoupdate_policy'] ?: array();
+    $policy_key = $version . '/' . $tier_id;
+    $poldef_key = $version . '/*';
+
+    if(isset($policy_map[$policy_key])) {
+        $value = $policy_map[$policy_key];
+    } elseif(isset($policy_map[$poldef_key])) {
+        $value = $policy_map[$poldef_key];
+    } else {
+        $value = 'default';
+    }
+
+    return $value;
+}
+
+function lookup_fixed($path, $snapshot_start) {
+    global $config;
+
+    $root_path = $config['snapshots_dir'] ?: "/srv/porthos/webroot/";
+    $snapshots = array_map('basename', glob($root_path . "d20*"));
+
+    $start_key = array_search($snapshot_start, $snapshots);
+    if( ! $start_key) {
+        // the fixed snapshot does not exist: fall back to the empty repository.
+        return 'empty';
+    }
+    $snapshots = array_slice($snapshots, $start_key);
+    foreach($snapshots as $snapshot) {
+        if(is_file($root_path . $snapshot . '/' . $path)) {
+            return $snapshot;
+        }
+    }
+    return 'head';
+}
+
+function lookup_monday($path, $tier_id) {
+    global $config;
+
+    $week_size = $config['week_size'] ?: 5;
+    $root_path = $config['snapshots_dir'] ?: "/srv/porthos/webroot/";
     $snapshots = array_reverse(array_map('basename', glob($root_path . "d20*")));
     $last_snapshot_day_id = date('w', get_snapshot_timestamp($snapshots[0]));
     // $monday_offset formula:
@@ -93,4 +134,19 @@ function lookup_snapshot($path, $tier_id = 0, $week_size = 5) {
         }
     }
     return $i < 0 ? 'head' : $snapshots[$i];
-}
+}
+
+function lookup_snapshot($path, $version, $tier_id) {
+    global $config;
+
+    $policy_name = lookup_policy($version, $tier_id);
+    if($policy_name == 'default') {
+        return lookup_monday($path, $tier_id);
+    } elseif(substr($policy_name, 0, 6) == 'fixed/') {
+        return lookup_fixed($path, substr($policy_name, 6));
+    } elseif($policy_name == 'empty') {
+        return 'empty';
+    } else {
+        return 'head';
+    }
+}

porthos/root/srv/porthos/script/mirrorlist.php

@@ -52,9 +52,5 @@
 }
 
 foreach($config['base_urls'] as $baseurl) {
-    if($use_tier && in_array($version, $config['stop_autoupdate'])) {
-        echo $baseurl . "empty\n";
-    } else {
-        echo $baseurl . $path . "${version}/${repo}/${arch}\n";
-    }
+    echo $baseurl . $path . "${version}/${repo}/${arch}\n";
 }

porthos/root/srv/porthos/test/tier-assignments.php

@@ -0,0 +1,47 @@
+<?php
+
+/*
+ * Copyright (C) 2019 Nethesis S.r.l.
+ * http://www.nethesis.it - nethserver@nethesis.it
+ *
+ * This script is part of Dartagnan.
+ *
+ * Dartagnan is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License,
+ * or any later version.
+ *
+ * Dartagnan is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Dartagnan.  If not, see COPYING.
+ */
+
+//
+// Test the automatic tier assignment procedure.
+//
+// 1. Read a list of system_id keys from standard input
+// 2. Print the system_id with resulting tier_id
+//
+
+while($system_id = trim(fgets(STDIN))) {
+
+    $hash = 0;
+
+    foreach(str_split($system_id) as $c) {
+        $hash += ord($c);
+    }
+    $hash = $hash % 10;
+    if($hash < 1) { // 10%
+        $tier_id = 0;
+    } elseif($hash < 3) { // +20% = 30%
+        $tier_id = 1;
+    } else { // +70% = 100%
+        $tier_id = 2;
+    }
+
+    echo $system_id . ' ' . $tier_id . "\n";
+}

porthos/root/srv/porthos/script/calc-tierid.php renamed to porthos/root/srv/porthos/test/tier-hits.php

@@ -20,6 +20,13 @@
  * along with Dartagnan.  If not, see COPYING.
  */
 
+//
+// Test the automatic tier assignment procedure.
+//
+// 1. Read a list of system_id keys from standard input
+// 2. Print the tiers hit percentages
+//
+
 $tier_hits = [0,0,0];
 
 while($system_id = trim(fgets(STDIN))) {
@@ -29,10 +36,10 @@
     foreach(str_split($system_id) as $c) {
         $hash += ord($c);
     }
-    $hash = $hash % 256;
-    if($hash < 26) { // 10%
+    $hash = $hash % 10;
+    if($hash < 1) { // 10%
         $tier_id = 0;
-    } elseif($hash < 77) { // +20% = 30%
+    } elseif($hash < 3) { // +20% = 30%
         $tier_id = 1;
     } else { // +70% = 100%
         $tier_id = 2;
@@ -44,7 +51,7 @@
 
 $total_hits = array_sum($tier_hits);
 
-printf("Total: %d\n", $total_hits);
+printf("Total hits: %d\n", $total_hits);
 foreach($tier_hits as $tier_id => $hits) {
-    printf("Tier %d, hits %4d - %.2f\n", $tier_id, $hits, 100*$hits/$total_hits);
+    printf("- tier %d: %2.2f%% (%d)\n", $tier_id, 100*$hits/$total_hits, $hits);
 }