Refactor assemblers out into decorated functions

dishpylib/README.md

@@ -0,0 +1,2 @@
+Library of Python code used throughout scripts.
+Python package requirements, listed in `third-party/requirements.txt`, are pre-installed in the project's Docker container.

dishpylib/pyassemblers/__init__.py

@@ -0,0 +1,26 @@
+from .assemble_config_records import assemble_config_records
+from .assemble_deletion_mutant_competitions import assemble_deletion_mutant_competitions
+from .assemble_deletion_mutant_phenotype_differentiation import assemble_deletion_mutant_phenotype_differentiation
+from .assemble_insertion_mutant_competitions import assemble_insertion_mutant_competitions
+from .assemble_insertion_mutant_phenotype_differentiation import assemble_insertion_mutant_phenotype_differentiation
+from .assemble_monoculture_dpp_metrics import assemble_monoculture_dpp_metrics
+from .assemble_monoculture_kin_conflict_by_replev_statistics import assemble_monoculture_kin_conflict_by_replev_statistics
+from .assemble_monoculture_kin_conflict_statistics import assemble_monoculture_kin_conflict_statistics
+from .assemble_mutant_competitions import assemble_mutant_competitions
+from .assemble_mutant_phenotype_differentiation import assemble_mutant_phenotype_differentiation
+from .assemble_mutating_competitions import assemble_mutating_competitions
+from .assemble_noncritical_nopout_fitness_competitions import assemble_noncritical_nopout_fitness_competitions
+from .assemble_phenotype_neutral_nopout_fitness_competitions import assemble_phenotype_neutral_nopout_fitness_competitions
+from .assemble_phenotype_neutral_nopout_phenotype_differentiation import assemble_phenotype_neutral_nopout_phenotype_differentiation
+from .assemble_phenotype_neutral_nopouts import assemble_phenotype_neutral_nopouts
+from .assemble_point_mutant_competitions import assemble_point_mutant_competitions
+from .assemble_point_mutant_phenotype_differentiation import assemble_point_mutant_phenotype_differentiation
+from .assemble_predecessor_battles import assemble_predecessor_battles
+from .assemble_predecessor_competitions import assemble_predecessor_competitions
+from .assemble_progenitor_competitions import assemble_progenitor_competitions
+from .assemble_strain_competitions import assemble_strain_competitions
+from .assemble_variant_competitions import assemble_variant_competitions
+from .assemble_wildtype_doubling_time import assemble_wildtype_doubling_time
+from .either_perturbation_competitions_assembler_factory import either_perturbation_competitions_assembler_factory
+from .messaging_selfsend_competitions_assembler_factory import messaging_selfsend_competitions_assembler_factory
+from .perturbation_competitions_assembler_factory import perturbation_competitions_assembler_factory

dishpylib/pyassemblers/assemble_config_records.py

@@ -0,0 +1,36 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pyhelpers import get_control_t_distns
+from dishpylib.pytabulators import \
+    tabulate_phenotype_equivalent_nopout_fitness
+
+@try_except_missing_data_decorator_factory('config records')
+@announce_job_decorator_factory('config records')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_config_records( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    config_records, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/evolve/config-records/stage=1+what=collated/stint={stint}/',
+    )
+
+    config_records_df = pd.read_csv(
+        f's3://{bucket}/{config_records.key}',
+    )
+
+    config_records_df['Series'] = config_records_df['SERIES']
+    config_records_df['Stint'] = config_records_df['STINT']
+
+    res_df = config_records_df
+    res_sources = [
+        config_records.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_deletion_mutant_competitions.py

@@ -0,0 +1,36 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pyhelpers import get_control_t_distns
+from dishpylib.pytabulators import tabulate_mutant_fitness
+
+@try_except_missing_data_decorator_factory('deletion mutant competitions')
+@announce_job_decorator_factory('deletion mutant competitions')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_deletion_mutant_competitions( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    mutant_competitions, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/mutant-deletion-competitions/stage=2+what=collated/stint={stint}/',
+    )
+
+    mutant_df = pd.read_csv(
+        f's3://{bucket}/{mutant_competitions.key}',
+    )
+
+    res_df = tabulate_mutant_fitness(
+        mutant_df,
+        get_control_t_distns(bucket, endeavor, stint),
+        'Deletion ',
+    )
+    res_sources = [
+        mutant_competitions.key
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_deletion_mutant_phenotype_differentiation.py

@@ -0,0 +1,35 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pytabulators import tabulate_mutant_phenotype_differentiation
+
+@try_except_missing_data_decorator_factory('mutant phenotype-differentiation')
+@announce_job_decorator_factory('mutant phenotype-differentiation')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_deletion_mutant_phenotype_differentiation(
+    *, bucket, endeavor, stint
+):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    mutant_phenotype_differentiation, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/mutant-deletion-phenotype-differentiation/stage=2+what=collated/stint={stint}/',
+    )
+
+    mutant_df = pd.read_csv(
+        f's3://{bucket}/{mutant_phenotype_differentiation.key}',
+    )
+
+    res_df = tabulate_mutant_phenotype_differentiation(
+        mutant_df, 'Deletion ',
+    )
+    res_sources = [
+        mutant_phenotype_differentiation.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_insertion_mutant_competitions.py

@@ -0,0 +1,36 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pyhelpers import get_control_t_distns
+from dishpylib.pytabulators import tabulate_mutant_fitness
+
+@try_except_missing_data_decorator_factory('insertion mutant competitions')
+@announce_job_decorator_factory('insertion mutant competitions')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_insertion_mutant_competitions( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    mutant_competitions, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/mutant-insertion-competitions/stage=2+what=collated/stint={stint}/',
+    )
+
+    mutant_df = pd.read_csv(
+        f's3://{bucket}/{mutant_competitions.key}',
+    )
+
+    res_df = tabulate_mutant_fitness(
+        mutant_df,
+        get_control_t_distns( bucket, endeavor, stint ),
+        'Insertion ',
+    )
+    res_sources = [
+        mutant_competitions.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_insertion_mutant_phenotype_differentiation.py

@@ -0,0 +1,39 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pytabulators import tabulate_mutant_phenotype_differentiation
+
+@try_except_missing_data_decorator_factory(
+    'insertion mutant phenotype-differentiation'
+)
+@announce_job_decorator_factory(
+    'insertion mutant phenotype-differentiation'
+)
+@only_every_nth_stint_decorator_factory(10)
+def assemble_insertion_mutant_phenotype_differentiation(
+    *, bucket, endeavor, stint
+):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    mutant_phenotype_differentiation, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/mutant-insertion-phenotype-differentiation/stage=2+what=collated/stint={stint}/',
+    )
+
+    mutant_df = pd.read_csv(
+        f's3://{bucket}/{mutant_phenotype_differentiation.key}',
+    )
+
+    res_df = tabulate_mutant_phenotype_differentiation(
+        mutant_df, 'Insertion ',
+    )
+    res_sources = [
+        mutant_phenotype_differentiation.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_monoculture_dpp_metrics.py

@@ -0,0 +1,31 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pytabulators import reshape_dpp
+
+@try_except_missing_data_decorator_factory('monoculture dpp metrics')
+@announce_job_decorator_factory('monoculture dpp metrics')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_monoculture_dpp_metrics( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    monoculture_dpp_metrics, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/monocultures/metrics/stage=2+what=collated/stint={stint}/a=demographic_phenotypic_phylogenetic_metrics+',
+    )
+
+    monoculture_dpp_df = pd.read_csv(
+        f's3://{bucket}/{monoculture_dpp_metrics.key}',
+    )
+
+    res_df = reshape_dpp( monoculture_dpp_df )
+    res_sources = [
+        monoculture_dpp_metrics.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_monoculture_kin_conflict_by_replev_statistics.py

@@ -0,0 +1,38 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pytabulators import reshape_kin_conflict_by_replev
+
+@try_except_missing_data_decorator_factory(
+    'monoculture kin conflict by replev statistics'
+)
+@announce_job_decorator_factory('monoculture kin conflict by replev statistics')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_monoculture_kin_conflict_by_replev_statistics(
+    *, bucket, endeavor, stint
+):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    monoculture_kin_conflict_by_replev_statistics, \
+        = bucket_handle.objects.filter(
+            Prefix=f'endeavor={endeavor}/monocultures/metrics/stage=2+what=collated/stint={stint}/a=kin_conflict_by_replev_statistics+',
+        )
+
+    monoculture_kin_conflict_by_replev_df = pd.read_csv(
+        f's3://{bucket}/{monoculture_kin_conflict_by_replev_statistics.key}',
+    )
+
+    res_df = reshape_kin_conflict_by_replev(
+        monoculture_kin_conflict_by_replev_df
+    )
+    res_sources = [
+        monoculture_kin_conflict_by_replev_statistics.key
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_monoculture_kin_conflict_statistics.py

@@ -0,0 +1,33 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pytabulators import reshape_kin_conflict
+
+@try_except_missing_data_decorator_factory(
+    'monoculture kin conflict statistics'
+)
+@announce_job_decorator_factory('monoculture kin conflict statistics')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_monoculture_kin_conflict_statistics( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    monoculture_kin_conflict_statistics, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/monocultures/metrics/stage=2+what=collated/stint={stint}/a=kin_conflict_statistics+'
+    )
+
+    monoculture_kin_conflict_df = pd.read_csv(
+        f's3://{bucket}/{monoculture_kin_conflict_statistics.key}'
+    )
+
+    res_df = reshape_kin_conflict( monoculture_kin_conflict_df )
+    res_sources = [
+        monoculture_kin_conflict_statistics.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_mutant_competitions.py

@@ -0,0 +1,35 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pyhelpers import get_control_t_distns
+from dishpylib.pytabulators import tabulate_mutant_fitness
+
+@try_except_missing_data_decorator_factory('mutant competitions')
+@announce_job_decorator_factory('mutant competitions')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_mutant_competitions( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    mutant_competitions, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/mutant-competitions/stage=2+what=collated/stint={stint}/',
+    )
+
+    mutant_df = pd.read_csv(
+        f's3://{bucket}/{mutant_competitions.key}',
+    )
+
+    res_df = tabulate_mutant_fitness(
+        mutant_df,
+        get_control_t_distns( bucket, endeavor, stint ),
+    )
+    res_sources = [
+        mutant_competitions.key,
+    ]
+
+    return res_df, res_sources

dishpylib/pyassemblers/assemble_mutant_phenotype_differentiation.py

@@ -0,0 +1,33 @@
+import boto3
+import pandas as pd
+
+from dishpylib.pydecorators import \
+    announce_job_decorator_factory, \
+    only_every_nth_stint_decorator_factory, \
+    try_except_missing_data_decorator_factory
+from dishpylib.pytabulators import tabulate_mutant_phenotype_differentiation
+
+@try_except_missing_data_decorator_factory('mutant phenotype-differentiation')
+@announce_job_decorator_factory('mutant phenotype-differentiation')
+@only_every_nth_stint_decorator_factory(10)
+def assemble_mutant_phenotype_differentiation( *, bucket, endeavor, stint ):
+
+    s3_handle = boto3.resource('s3')
+    bucket_handle = s3_handle.Bucket(bucket)
+
+    mutant_phenotype_differentiation, = bucket_handle.objects.filter(
+        Prefix=f'endeavor={endeavor}/mutant-phenotype-differentiation/stage=2+what=collated/stint={stint}/',
+    )
+
+    mutant_df = pd.read_csv(
+        f's3://{bucket}/{mutant_phenotype_differentiation.key}',
+    )
+
+    res_df = tabulate_mutant_phenotype_differentiation(
+        mutant_df,
+    )
+    res_sources = [
+        mutant_phenotype_differentiation.key,
+    ]
+
+    return res_df, res_sources