New branch pg tran

client/benchmarks/pgbench.py

@@ -214,7 +214,7 @@ def run_tests(self, csv_queue):
         # derive configuration for the CPU count / RAM size
         configs = PgBench._configure(cpu_count(), available_ram())
 
-        results = {'ro': {}, 'rw': {}}
+        results = {'ro': {}, 'rw': {}}  #ro:read only  rw:read-write
         j = 0
         for config in configs:
             scale = config['scale']

client/benchmarks/runner.py

@@ -83,7 +83,8 @@ def _run_config(self, config_name):
         # expand the attribute names
         bench = bench(**config['config'])
 
-        self._cluster.start(config=config['postgres'])
+        self._cluster.start(
+            config=config['postgres'])  # obtain the database name pgperffarm-db
 
         # start collector(s) of additional info
         self._collector.start()
@@ -116,16 +117,16 @@ def _run_config(self, config_name):
         uname = check_output(['uname', '-a'])
 
         r['meta'] = {
-                'benchmark': config['benchmark'],
-                'date': strftime("%Y-%m-%d %H:%M:%S.000000+00", gmtime()),
-                'name': config_name,
-                'uname': uname,
+            'benchmark': config['benchmark'],
+            'date': strftime("%Y-%m-%d %H:%M:%S.000000+00", gmtime()),
+            'name': config_name,
+            'uname': uname,
         }
 
         r['postgres'] = {
-                'branch': config['branch'],
-                'commit': config['commit'],
-                'settings': config['postgres'],
+            'branch': config['branch'],
+            'commit': config['commit'],
+            'settings': config['postgres'],
         }
 
         with open('%s/results.json' % self._output, 'w') as f:
@@ -134,23 +135,23 @@ def _run_config(self, config_name):
         try:
             self._upload_results(r)
         except Exception as e:
-            print (e)
+            print(e)
 
     def _upload_results(self, results):
 
-        postdata = results  
-        post = []  
+        postdata = results
+        post = []
         post.append(postdata)
 
         headers = {'Content-Type': 'application/json; charset=utf-8', 'Authorization': self._secret}
+        # r = requests.post(self._url+"upload/".encode('utf-8'), data=json.dumps(post).encode('utf-8'), headers=headers)
         r = requests.post(self._url.encode('utf-8'), data=json.dumps(post).encode('utf-8'), headers=headers)
 
-
     def run(self):
         'run all the configured benchmarks'
 
         # Removing the existing directory
-        
+
         try:
             os.mkdir(self._output)
         except OSError as e:

client/collectors/collectd.py

@@ -3,8 +3,8 @@
 from utils.logging import log
 from utils.misc import run_cmd
 
-COLLECTD_CONFIG = '/tmp/.collectd.conf'
-COLLECTD_PIDFILE = '/tmp/.collectd.pid'
+COLLECTD_CONFIG = '/raid/.collectd.conf'
+COLLECTD_PIDFILE = '/raid/.collectd.pid'
 
 
 class CollectdCollector(object):

client/perffarm-client.py

@@ -19,8 +19,9 @@
 
 from settings_local import *
 from settings import *
-API_URL = 'http://127.0.0.1:8000/'
-MACHINE_SECRET = '610f79063e62e6ad09460ac2c4e66da0386dc89b'
+# API_URL = 'http://127.0.0.1:8000/'
+API_URL = 'http://127.0.0.1:8000/upload/'
+MACHINE_SECRET = 'e984c3017cd1a0dff0ef9f0c394a5c285e421411' #This machine secrete is specific to my environment
 if __name__ == '__main__':
 
     with FileLock('.lock') as lock:

client/settings.py

@@ -2,11 +2,11 @@
 import sys
 
 # global configuration
-GIT_URL = 'https://github.com/postgres/postgres.git'
-REPOSITORY_PATH = '/tmp/git-postgres'
-BUILD_PATH = '/Users/chenzhang/anaconda3'
+GIT_URL = 'https://gitee.com/purpleyu/postgres.git' #changed to a local repo
+REPOSITORY_PATH = '/raid/git-postgres/postgres'
+BUILD_PATH = '/usr/lib/postgresql/11'
 BIN_PATH = os.path.join(BUILD_PATH, 'bin')
-DATADIR_PATH = '/tmp/data-postgres'
+DATADIR_PATH = '/raid/data-postgres/postgres'
 
 POSTGRES_CONFIG = {
     'shared_buffers': '1GB',
@@ -22,9 +22,9 @@
     'checkpoint_completion_target': '0.9',
 }
 
-DATABASE_NAME = 'postgres' # This name needs to be the same as rest_api settings_local.py database NAME
+DATABASE_NAME = 'pgperffarm-db' # This name needs to be the same as rest_api settings_local.py database NAME
 
-OUTPUT_DIR = '/tmp/perf-output'
+OUTPUT_DIR = '/raid/perf-output'
 
 # configuration for PgBench
 # runs - number of repetitions (including test for all client counts)

rest_api/apps/machines/migrations/0001_initial.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-03 17:22
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Alias',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=32, unique=True, verbose_name='alias name')),
+                ('is_used', models.BooleanField(default=False, verbose_name='is_used')),
+                ('add_time', models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Machine',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('add_time', models.DateTimeField(auto_now_add=True)),
+                ('alias', models.CharField(blank=True, default='', max_length=100)),
+                ('machine_secret', models.CharField(blank=True, default='', max_length=32, verbose_name='machine secret')),
+                ('sn', models.CharField(blank=True, default='', max_length=16)),
+                ('os_name', models.CharField(blank=True, default='', max_length=100)),
+                ('os_version', models.CharField(blank=True, default='', max_length=100)),
+                ('comp_name', models.CharField(blank=True, default='', max_length=100)),
+                ('comp_version', models.CharField(blank=True, default='', max_length=100)),
+                ('state', models.CharField(choices=[('A', 'Active'), ('I', 'Inactive')], default='A', max_length=10)),
+                ('owner_email', models.EmailField(max_length=256, verbose_name='email')),
+                ('owner_username', models.CharField(blank=True, default='', max_length=100)),
+                ('owner_id', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='machines', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ('add_time',),
+            },
+        ),
+    ]

rest_api/apps/machines/migrations/0002_auto_20200213_1523.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-13 15:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('machines', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='machine',
+            name='machine_secret',
+            field=models.CharField(blank=True, default='', max_length=100, verbose_name='machine secret'),
+        ),
+    ]

rest_api/apps/machines/models.py

@@ -29,7 +29,7 @@ class Machine(models.Model):
 
 	add_time = models.DateTimeField(auto_now_add=True)
 	alias = models.CharField(max_length=100, blank=True, default='')
-	machine_secret = models.CharField(max_length=32, blank=True, default='', verbose_name="machine secret")
+	machine_secret = models.CharField(max_length=100, blank=True, default='', verbose_name="machine secret")
 	sn = models.CharField(max_length=16, blank=True, default='')
 	os_name = models.CharField(max_length=100, blank=True, default='')
 	os_version = models.CharField(max_length=100, blank=True, default='')

rest_api/apps/records/migrations/0001_initial.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-03 17:22
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('machines', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LinuxInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('mounts', models.TextField(default='null', help_text='mounts', verbose_name='mounts')),
+                ('cpuinfo', models.TextField(default='null', help_text='cpuinfo', verbose_name='cpuinfo')),
+                ('sysctl', models.TextField(default='null', help_text='sysctl', verbose_name='sysctl')),
+                ('meminfo', models.TextField(default='null', help_text='meminfo', verbose_name='meminfo')),
+            ],
+            options={
+                'verbose_name': 'linux info',
+                'verbose_name_plural': 'linux info',
+            },
+        ),
+        migrations.CreateModel(
+            name='MetaInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('date', models.DateTimeField(help_text='date', verbose_name='date')),
+                ('uname', models.TextField(help_text='uname', verbose_name='uname')),
+                ('benchmark', models.TextField(help_text='benchmark', verbose_name='benchmark')),
+                ('name', models.TextField(help_text='name', verbose_name='name')),
+            ],
+            options={
+                'verbose_name': 'meta info',
+                'verbose_name_plural': 'meta info',
+            },
+        ),
+        migrations.CreateModel(
+            name='PGInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('checkpoint_timeout', models.CharField(help_text='checkpoint_timeout', max_length=32, verbose_name='checkpoint_timeout')),
+                ('log_temp_files', models.IntegerField(help_text='log_temp_files', verbose_name='log_temp_files')),
+                ('work_mem', models.CharField(help_text='work_mem', max_length=32, verbose_name='work_mem')),
+                ('log_line_prefix', models.CharField(help_text='checkpoint_timeout', max_length=64, verbose_name='checkpoint_timeout')),
+                ('shared_buffers', models.CharField(help_text='shared_buffers', max_length=32, verbose_name='shared_buffers')),
+                ('log_autovacuum_min_duration', models.IntegerField(help_text='log_autovacuum_min_duration', verbose_name='log_autovacuum_min_duration')),
+                ('checkpoint_completion_target', models.DecimalField(decimal_places=4, help_text='checkpoint_completion_target', max_digits=8, verbose_name='checkpoint_completion_target')),
+                ('maintenance_work_mem', models.CharField(help_text='maintenance_work_mem', max_length=32, verbose_name='maintenance_work_mem')),
+                ('log_checkpoints', models.IntegerField(choices=[(1, 'on'), (2, 'off')], help_text='log_checkpoints', verbose_name='log_checkpoints')),
+                ('max_wal_size', models.CharField(help_text='max_wal_size', max_length=32, verbose_name='max_wal_size')),
+                ('min_wal_size', models.CharField(help_text='min_wal_size', max_length=32, verbose_name='min_wal_size')),
+            ],
+            options={
+                'verbose_name': 'pg info',
+                'verbose_name_plural': 'pg info',
+            },
+        ),
+        migrations.CreateModel(
+            name='TestBranch',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('branch_name', models.CharField(help_text='branch name', max_length=128, unique=True, verbose_name='branch name')),
+                ('branch_order', models.IntegerField(default=5, help_text='order in all the  branch', verbose_name='branch order')),
+                ('is_show', models.BooleanField(default=True, help_text='branch isshow', verbose_name='branch is shown')),
+                ('is_accept', models.BooleanField(default=True, help_text='branch accepts new reports', verbose_name='branch accepts new reports')),
+                ('add_time', models.DateTimeField(default=django.utils.timezone.now, help_text='branch added time', verbose_name='branch added time')),
+            ],
+            options={
+                'verbose_name': 'test branch',
+                'verbose_name_plural': 'test branch',
+            },
+        ),
+        migrations.CreateModel(
+            name='TestCategory',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('cate_name', models.CharField(help_text='cate name', max_length=64, verbose_name='cate name')),
+                ('cate_sn', models.CharField(help_text='cate sn', max_length=32, unique=True, verbose_name='cate sn')),
+                ('cate_order', models.IntegerField(help_text='order in the current level', verbose_name='cate order')),
+                ('add_time', models.DateTimeField(default=django.utils.timezone.now, help_text='category added time', verbose_name='add time')),
+            ],
+            options={
+                'verbose_name': 'tests category',
+                'verbose_name_plural': 'tests category',
+            },
+        ),
+        migrations.CreateModel(
+            name='TestDataSet',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('clients', models.IntegerField(help_text='clients of the test dataset', verbose_name='clients')),
+                ('scale', models.IntegerField(help_text='scale of the test dataset', verbose_name='scale')),
+                ('std', models.DecimalField(decimal_places=8, help_text='std of the test dataset', max_digits=18, verbose_name='std')),
+                ('metric', models.DecimalField(decimal_places=8, help_text='metric of the test dataset', max_digits=18, verbose_name='metric')),
+                ('median', models.DecimalField(decimal_places=8, help_text='median of the test dataset', max_digits=18, verbose_name='median')),
+                ('status', models.IntegerField(choices=[(-1, 'none'), (1, 'improved'), (2, 'quo'), (3, 'regressive')], help_text='status of this dataset', verbose_name='status')),
+                ('percentage', models.DecimalField(decimal_places=4, help_text='percentage compared to previous dataset', max_digits=8, verbose_name='percentage')),
+                ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test dataset time')),
+                ('prev', models.ForeignKey(blank=True, help_text='previous test dataset id', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='prev1', to='records.TestDataSet', verbose_name='previous test dataset id')),
+                ('test_cate', models.ForeignKey(help_text='test cate id', on_delete=django.db.models.deletion.CASCADE, to='records.TestCategory', verbose_name='test cate id')),
+            ],
+            options={
+                'verbose_name': 'test dataset',
+                'verbose_name_plural': 'test dataset',
+            },
+        ),
+        migrations.CreateModel(
+            name='TestRecord',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('test_desc', models.TextField(help_text='test desc', verbose_name='test desc')),
+                ('meta_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='meta time')),
+                ('hash', models.CharField(default='', help_text='record hash', max_length=128, unique=True, verbose_name='record hash')),
+                ('uuid', models.CharField(default='', help_text='record uuid', max_length=64, unique=True, verbose_name='record uuid')),
+                ('commit', models.CharField(help_text='record commit', max_length=64, verbose_name='record commit')),
+                ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test added time')),
+                ('branch', models.ForeignKey(help_text='pg branch', on_delete=django.db.models.deletion.CASCADE, to='records.TestBranch', verbose_name='pg branch')),
+                ('linux_info', models.ForeignKey(help_text='linux info', on_delete=django.db.models.deletion.CASCADE, to='records.LinuxInfo', verbose_name='linux info')),
+                ('meta_info', models.ForeignKey(help_text='meta info', on_delete=django.db.models.deletion.CASCADE, to='records.MetaInfo', verbose_name='meta info')),
+                ('pg_info', models.ForeignKey(help_text='pg info', on_delete=django.db.models.deletion.CASCADE, to='records.PGInfo', verbose_name='pg info')),
+                ('test_machine', models.ForeignKey(help_text='person who add this test item', on_delete=django.db.models.deletion.CASCADE, to='machines.Machine', verbose_name='test owner')),
+            ],
+            options={
+                'verbose_name': 'tests',
+                'verbose_name_plural': 'tests',
+            },
+        ),
+        migrations.CreateModel(
+            name='TestResult',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('latency', models.IntegerField(help_text='latency of the test result', verbose_name='latency')),
+                ('scale', models.IntegerField(help_text='scale of the test result', verbose_name='scale')),
+                ('end', models.DecimalField(decimal_places=12, help_text='endtime of the test result', max_digits=32, verbose_name='end')),
+                ('clients', models.IntegerField(help_text='clients of the test result', verbose_name='clients')),
+                ('start', models.DecimalField(decimal_places=12, help_text='starttime of the test result', max_digits=32, verbose_name='start')),
+                ('tps', models.DecimalField(decimal_places=6, default=0, help_text='tps of the test result', max_digits=18, verbose_name='tps')),
+                ('run', models.IntegerField(help_text='run number', verbose_name='run')),
+                ('threads', models.IntegerField(help_text='threads of the test result', verbose_name='threads')),
+                ('mode', models.IntegerField(choices=[(1, 'simple'), (2, 'other'), (-1, 'test')], help_text='test mode', verbose_name='mode')),
+                ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test result added time')),
+                ('test_dataset', models.ForeignKey(help_text='test dataset id', on_delete=django.db.models.deletion.CASCADE, to='records.TestDataSet', verbose_name='test dataset id')),
+            ],
+            options={
+                'verbose_name': 'test result',
+                'verbose_name_plural': 'test result',
+            },
+        ),
+        migrations.AddField(
+            model_name='testdataset',
+            name='test_record',
+            field=models.ForeignKey(help_text='test record id', on_delete=django.db.models.deletion.CASCADE, to='records.TestRecord', verbose_name='test record id'),
+        ),
+    ]

rest_api/apps/records/migrations/0002_auto_20200215_1749.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-15 17:49
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('records', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='testrecord',
+            name='commit',
+            field=models.CharField(help_text='record commit', max_length=100, verbose_name='record commit'),
+        ),
+    ]