Bump version (#4)

* Pin specific Celery/Kombu/Redis versions (google#305)

* in TurbiniaTaskResult, input_evidence doesn't have to be a list (google#309)

* input_evidence doesn't have to be a list

* Fix exception

* current pypi version of google-cloud-storage (1.13.0) requires google-cloud-core 0.28.1 (before the rename of google.cloud.iam (core) to google.api_core.iam (google#315)

* Use a link to a "parent Evidence" instead of subclassing (google#296)

* parent evidence

* undo some simplifications for the sake of a simpler CL

* Add some serialization

* update docstring

* Initialize attribute

* set parent evidence if Evidence type is context dependant

* don't pass parent_evidence at instantiation

* undo linter stuff

* comments

* fix aim lib breaking tests

* typo

* Print version on start 3 (google#320)

* Add files via upload

* Delete turbiniactl.py

* Delete turbiniactl.py

* Add files via upload

* Delete turbiniactl.py

* Add files via upload

* Update turbiniactl.py

* Caps

* Quick update to evidence docstrings (google#317)

... to disambiguate between _preprocess() and preprocess().

* Add Job filters (google#247)

* Add job filters

* fix docstrings.

* update docstring

* Get jobs filters working with new job manager

* Refactor out FilterJobObjects into new method

* Update YAPF

* remove missed confict markers

* Docstrings and renaming

* Migrate job graph generator to use new manager (google#321)

* Update Evidence local_path when it's saved (google#319)

* Pin google-cloud-storage to 1.13.0 (google#326)

Fixes google#325

Looks like google-cloud-storage was updated in:
googleapis/google-cloud-python#6741

Which just got released as 1.13.1:
https://pypi.org/project/google-cloud-storage/#history

* Set image export to process all partitions (google#324)

* Add --partitions all to image_export invocations

* Fix typo

* Explicitly set saved_paths to list (google#323)

* Move version print after log level setup (google#322)

* Move version print after log level setup

* Remove extra whitespace

* update the pystyle link (google#333)

* Undefined name: Define 'unicode' in Python 3 (google#337)

* Undefined name: Define 'unicode' in Python 3

__unicode()__ was removed in Python 3 because all __str__ are Unicode.

[flake8](http://flake8.pycqa.org) testing of https://github.com/google/turbinia on Python 3.7.1

$ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__
```
./tools/turbinia_job_graph.py:47:40: F821 undefined name 'unicode'
  parser.add_argument('filename', type=unicode, help='where to save the file')
                                       ^
1     F821 undefined name 'unicode'
1
```

* Placate PyLint

* Added PSQ timeout to 1 week (google#336)

* Error when worker version doesn't match server google#307 (google#327)

* Added turbina_version to TurbinaTask

* First approach

* Changed to no rise error and return instead

* Restored the run from run_wrapper

* Changed format of strings

* Changed words fixed line too long

* bump version

docs/contributing.md

@@ -69,7 +69,7 @@ that every developer knows to (largely) expect the same coding style.
 #### Style guide
 
 We primarily follow the
-[Google Python Style Guide](https://google-styleguide.googlecode.com/svn/trunk/pyguide.html).
+[Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
 Various Turbinia specific additions/variations are:
 
 *   Using two spaces instead of four

tools/turbinia_job_graph.py

@@ -19,7 +19,12 @@
 import argparse
 import graphviz
 
-from turbinia.jobs import get_jobs as turbinia_jobs
+from turbinia.jobs import manager as jobs_manager
+
+try:
+  unicode
+except NameError:
+  unicode = str  # pylint: disable=redefined-builtin
 
 
 def create_graph():
@@ -29,15 +34,15 @@ def create_graph():
     Instance of graphviz.dot.Digraph
   """
   dot = graphviz.Digraph(comment='Turbinia Evidence graph', format='png')
-  for job in turbinia_jobs():
-    dot.node(job.name)
+  for _, job in jobs_manager.JobsManager.GetJobs():
+    dot.node(job.NAME)
     for evidence in job.evidence_input:
       dot.node(evidence.__name__, shape='box')
-      dot.edge(evidence.__name__, job.name)
+      dot.edge(evidence.__name__, job.NAME)
 
     for evidence in job.evidence_output:
       dot.node(evidence.__name__, shape='box')
-      dot.edge(job.name, evidence.__name__)
+      dot.edge(job.NAME, evidence.__name__)
   return dot
 
 

turbinia/__init__.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Main Turbinia application."""
 
-__version__ = '20181004+fb20190103'
+__version__ = '20181004+fb20190109'
 
 
 class TurbiniaException(Exception):

turbinia/client.py

@@ -307,6 +307,8 @@ def format_task_status(
                 task.get('name'), task.get('user'), task.get('worker_name'),
                 success, status))
         saved_paths = task.get('saved_paths', [])
+        if saved_paths is None:
+          saved_paths = []
         for path in saved_paths:
           results.append('\t{0:s}'.format(path))
       else:
@@ -423,11 +425,16 @@ class TurbiniaServer(object):
     task_manager (TaskManager): An object to manage turbinia tasks.
   """
 
-  def __init__(self):
-    """Initialize Turbinia Server."""
+  def __init__(self, jobs_blacklist=None, jobs_whitelist=None):
+    """Initializes Turbinia Server.
+
+    Args:
+      jobs_blacklist (Optional[list[str]]): Jobs we will exclude from running
+      jobs_whitelist (Optional[list[str]]): The only Jobs we will include to run
+    """
     config.LoadConfig()
     self.task_manager = task_manager.get_task_manager()
-    self.task_manager.setup()
+    self.task_manager.setup(jobs_blacklist, jobs_whitelist)
 
   def start(self):
     """Start Turbinia Server."""

turbinia/evidence.py

@@ -31,7 +31,6 @@
 if config.TASK_MANAGER.lower() == 'psq':
   from turbinia.processors import google_cloud
 
-
 def evidence_decode(evidence_dict):
   """Decode JSON into appropriate Evidence object.
 
@@ -64,6 +63,8 @@ def evidence_decode(evidence_dict):
         'No Evidence object of type {0:s} in evidence module'.format(type_))
 
   evidence.__dict__ = evidence_dict
+  if evidence_dict['parent_evidence']:
+    evidence.parent_evidence = evidence_decode(evidence_dict['parent_evidence'])
   return evidence
 
 
@@ -78,6 +79,8 @@ class Evidence(object):
         processing this evidence.
     cloud_only: Set to True for evidence types that can only be processed in a
         cloud environment, e.g. GoogleCloudDisk.
+    context_dependent: Whether this evidence is required to be built upon the
+        context of a parent evidence.
     copyable: Whether this evidence can be copied.  This will be set to True for
         object types that we want to copy to/from storage (e.g. PlasoFile, but
         not RawDisk).
@@ -91,7 +94,10 @@ class Evidence(object):
         that created it, if appropriate).
     local_path: A string of the local_path to the evidence.
     tags: dict of extra tags associated with this evidence.
-    request_id: The id of the request this evidence came from, if any
+    request_id: The id of the request this evidence came from, if any.
+    parent_evidence: The Evidence object that was used to generate this one, and
+        which pre/post process methods we need to re-execute to access data
+        relevant to us.
   """
 
   def __init__(
@@ -100,12 +106,14 @@ def __init__(
     """Initialization for Evidence."""
     self.copyable = False
     self.config = {}
+    self.context_dependent = False
     self.cloud_only = False
     self.description = description
     self.source = source
     self.local_path = local_path
     self.tags = tags if tags else {}
     self.request_id = request_id
+    self.parent_evidence = None
 
     # List of jobs that have processed this evidence
     self.processed_by = []
@@ -122,7 +130,10 @@ def __repr__(self):
 
   def serialize(self):
     """Return JSON serializable object."""
-    return self.__dict__
+    serialized_evidence = self.__dict__
+    if self.parent_evidence:
+      serialized_evidence['parent_evidence'] = self.parent_evidence.serialize()
+    return serialized_evidence
 
   def to_json(self):
     """Convert object to JSON.
@@ -142,7 +153,7 @@ def to_json(self):
 
     return serialized
 
-  def preprocess(self):
+  def _preprocess(self):
     """Preprocess this evidence prior to task running.
 
     This gets run in the context of the local task execution on the worker
@@ -151,7 +162,7 @@ def preprocess(self):
     """
     pass
 
-  def postprocess(self):
+  def _postprocess(self):
     """Postprocess this evidence after the task runs.
 
     This gets run in the context of the local task execution on the worker
@@ -160,6 +171,28 @@ def postprocess(self):
     """
     pass
 
+  def preprocess(self):
+    """Runs the possible parent's evidence preprocessing code, then ours.
+
+    This is a wrapper function that will call the chain of pre-processors
+    starting with the most distant ancestor.  After all of the ancestors have
+    been processed, then we run our pre-processor.
+    """
+    if self.parent_evidence:
+      self.parent_evidence.preprocess()
+    self._preprocess()
+
+  def postprocess(self):
+    """Runs our postprocessing code, then our possible parent's evidence.
+
+    This is is a wrapper function that will run our post-processor, and will
+    then recurse down the chain of parent Evidence and run those post-processors
+    in order.
+    """
+    self._postprocess()
+    if self.parent_evidence:
+      self.parent_evidence.postprocess()
+
 
 class Directory(Evidence):
   """Filesystem directory evidence."""
@@ -185,6 +218,13 @@ def __init__(
     self.size = size
     super(RawDisk, self).__init__(*args, **kwargs)
 
+  def _preprocess(self):
+    self.loopdevice_path = mount_local.PreprocessLosetup(self.local_path)
+
+  def _postprocess(self):
+    mount_local.PostprocessDeleteLosetup(self.loopdevice_path)
+    self.loopdevice_path = None
+
 
 class EncryptedDisk(RawDisk):
   """Encrypted disk file evidence.
@@ -224,10 +264,10 @@ def __init__(self, project=None, zone=None, disk_name=None, *args, **kwargs):
     super(GoogleCloudDisk, self).__init__(*args, **kwargs)
     self.cloud_only = True
 
-  def preprocess(self):
+  def _preprocess(self):
     self.local_path = google_cloud.PreprocessAttachDisk(self.disk_name)
 
-  def postprocess(self):
+  def _postprocess(self):
     google_cloud.PostprocessDetachDisk(self.disk_name, self.local_path)
     self.local_path = None
 
@@ -249,14 +289,14 @@ def __init__(self, embedded_path=None, *args, **kwargs):
     self.embedded_path = embedded_path
     super(GoogleCloudDiskRawEmbedded, self).__init__(*args, **kwargs)
 
-  def preprocess(self):
+  def _preprocess(self):
     self.local_path = google_cloud.PreprocessAttachDisk(self.disk_name)
     self.loopdevice_path = mount_local.PreprocessLosetup(self.local_path)
     self.mount_path = mount_local.PreprocessMountDisk(
         self.loopdevice_path, self.mount_partition)
     self.local_path = os.path.join(self.mount_path, self.embedded_path)
 
-  def postprocess(self):
+  def _postprocess(self):
     google_cloud.PostprocessDetachDisk(self.disk_name, self.local_path)
     mount_local.PostprocessUnmountPath(self.mount_path)
     mount_local.PostprocessDeleteLosetup(self.loopdevice_path)

turbinia/jobs/manager.py

@@ -16,12 +16,65 @@
 
 from __future__ import unicode_literals
 
+from turbinia import TurbiniaException
+
 
 class JobsManager(object):
   """The jobs manager."""
 
   _job_classes = {}
 
+  @classmethod
+  def FilterJobNames(cls, job_names, jobs_blacklist=None, jobs_whitelist=None):
+    """Filters a list of job names against white/black lists.
+
+    jobs_whitelist and jobs_blacklist must not be specified at the same time.
+
+    Args:
+      job_names (list[str]): The names of the job_names to filter.
+      jobs_blacklist (Optional[list[str]]): Job names to exclude.
+      jobs_whitelist (Optional[list[str]]): Job names to include.
+
+    Returns:
+     list[str]: Job names
+
+    Raises:
+      TurbiniaException if both jobs_blacklist and jobs_whitelist are specified.
+    """
+    jobs_blacklist = jobs_blacklist if jobs_blacklist else []
+    jobs_blacklist = [job.lower() for job in jobs_blacklist]
+    jobs_whitelist = jobs_whitelist if jobs_whitelist else []
+    jobs_whitelist = [job.lower() for job in jobs_whitelist]
+
+    if jobs_whitelist and jobs_blacklist:
+      raise TurbiniaException(
+          'jobs_whitelist and jobs_blacklist cannot be specified at the same '
+          'time.')
+    elif jobs_blacklist:
+      return [job for job in job_names if job.lower() not in jobs_blacklist]
+    elif jobs_whitelist:
+      return [job for job in job_names if job.lower() in jobs_whitelist]
+    else:
+      return job_names
+
+  @classmethod
+  def FilterJobObjects(cls, jobs, jobs_blacklist=None, jobs_whitelist=None):
+    """Filters a list of job objects against white/black lists.
+
+    jobs_whitelist and jobs_blacklist must not be specified at the same time.
+
+    Args:
+      jobs (list[TurbiniaJob]): The jobs to filter.
+      jobs_blacklist (Optional[list[str]]): Job names to exclude.
+      jobs_whitelist (Optional[list[str]]): Job names to include.
+
+    Returns:
+     list[TurbiniaJob]: Job objects
+    """
+    job_names = [job.name.lower() for job in jobs]
+    job_names = cls.FilterJobNames(job_names, jobs_blacklist, jobs_whitelist)
+    return [job for job in jobs if job.name.lower() in job_names]
+
   @classmethod
   def DeregisterJob(cls, job_class):
     """Deregisters a job class.

turbinia/jobs/manager_test.py

@@ -6,6 +6,7 @@
 
 import unittest
 
+from turbinia import TurbiniaException
 from turbinia.jobs import interface
 from turbinia.jobs import manager
 
@@ -94,6 +95,55 @@ def testGetJobInstances(self):
     for job in jobs:
       self.assertIsInstance(job, interface.TurbiniaJob)
 
+  def testFilterJobNamesEmptyLists(self):
+    """Test FilterJobNames() with no filters."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[], jobs_whitelist=[])
+    self.assertListEqual(job_names, return_job_names)
+
+  def testFilterJobNamesBlackList(self):
+    """Test FilterJobNames() with jobs_blacklist."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[job_names[0]], jobs_whitelist=[])
+    self.assertListEqual(job_names[1:], return_job_names)
+
+  def testFilterJobObjectsBlackList(self):
+    """Test FilterJobObjects() with jobs_blacklist and objects."""
+    jobs = [TestJob1(), TestJob2()]
+    return_jobs = manager.JobsManager.FilterJobObjects(
+        jobs, jobs_blacklist=[jobs[0].name], jobs_whitelist=[])
+    self.assertListEqual(jobs[1:], return_jobs)
+
+  def testFilterJobNamesWhiteList(self):
+    """Test FilterJobNames() with jobs_whitelist."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[], jobs_whitelist=[job_names[0]])
+    self.assertListEqual(job_names[:1], return_job_names)
+
+  def testFilterJobObjectsWhiteList(self):
+    """Test FilterJobObjects() with jobs_whitelist."""
+    jobs = [TestJob1(), TestJob2()]
+    return_jobs = manager.JobsManager.FilterJobObjects(
+        jobs, jobs_blacklist=[], jobs_whitelist=[jobs[1].name])
+    self.assertListEqual(jobs[1:], return_jobs)
+
+  def testFilterJobNamesException(self):
+    """Test FilterJobNames() with both jobs_blacklist and jobs_whitelist."""
+    job_names = ['testjob1', 'testjob2']
+    self.assertRaises(
+        TurbiniaException, manager.JobsManager.FilterJobNames, job_names,
+        jobs_blacklist=['a'], jobs_whitelist=['b'])
+
+  def testFilterJobNamesMixedCase(self):
+    """Test FilterJobNames() with mixed case inputs."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[], jobs_whitelist=['TESTJOB1'])
+    self.assertListEqual(job_names[:1], return_job_names)
+
 
 if __name__ == '__main__':
   unittest.main()

turbinia/lib/utils.py

@@ -40,7 +40,7 @@ def extract_artifacts(artifact_names, disk_path, output_dir):
 
   image_export_cmd = [
       'image_export.py', '--artifact_filters', artifacts, '--write', output_dir,
-      disk_path
+      '--partitions', 'all', disk_path
   ]
 
   # TODO: Consider break the exec helper to gather stdin/err.