Merge pull request #604 from 4dn-dcic/ajs_250707_skip_fq_runs_on_restricted_not_uploaded

Ajs 250707 skip fq runs on restricted not uploaded

Author: aschroed

CHANGELOG.rst

@@ -7,6 +7,12 @@ foursight
 Change Log
 ----------
 
+4.9.28
+==========
+* update fastq checks to not run pipelines on pre-release status files that have not been uploaded as OK as they will become restricted
+* md5run_uploaded, fastqc, fastq_first_line
+
+
 4.9.27
 ==========
 * modify missing raw file check to not add badge if files are restricted and missing as this is expected in some cases

chalicelib_fourfront/checks/helpers/wfr_utils.py

@@ -2072,3 +2072,19 @@ def limit_number_of_runs(check, my_auth):
         check.summary = f'Limiting the number of workflow runs to {n_runs_max} every 6h'
         check.full_output = {}
     return check, n_runs_available
+
+
+def prereleased_and_not_uploaded(connection, filemeta):
+    """Check if a file is pre-released and not uploaded yet.
+    This should only happen for files that are to be restricted status upon release.
+    Some files with pipelines will be uploaded so should have the usual basic QC run on them,
+    however, there will be cases where there is no processing pipeline so no need to upload the 
+    raw files and so the md5, fastq_first_line and fastQC can be skipped"""
+    statuses_to_check = ['pre-release', 'restricted']
+    if filemeta.get('status') not in statuses_to_check:
+        return False
+    my_s3_util = s3Utils(env=connection.ff_env)
+    if not my_s3_util.does_key_exist(filemeta.get('upload_key'), my_s3_util.raw_file_bucket, False):
+        # if the file is not in the raw bucket, it is not uploaded
+        return True
+    return False

chalicelib_fourfront/checks/wfr_checks.py

@@ -161,8 +161,6 @@ def md5run_status(connection, **kwargs):
             check.brief_output.append('did not complete checking all')
             break
 
-        # cnt of files to be triggered at this iteration of loop
-        n_runs_to_trigger = len(missing_md5_to_start + not_switched_status)
         # find bucket
         if 'FileProcessed' in a_file['@type']:
             my_bucket = out_bucket
@@ -256,12 +254,21 @@ def md5run_uploaded_files(connection, **kwargs):
     query = '/search/?type=File&md5sum=No+value' + ''.join(['&status=' + s for s in statuses])
 
     files = {}
-    files['uploaded_without_md5run'] = [f['accession'] for f in ff_utils.search_metadata(
-        query + '&workflow_run_inputs.workflow.title%21=md5+0.2.6', key=my_auth)]
-    files['uploaded_with_md5run'] = [f['accession'] for f in ff_utils.search_metadata(
-        query + '&workflow_run_inputs.workflow.title=md5+0.2.6', key=my_auth)]
+    no_md5_files = ff_utils.search_metadata(query, key=my_auth)
+    no_md5_files = [f for f in no_md5_files if not wfr_utils.prereleased_and_not_uploaded(connection, f)]
+    for f in no_md5_files:
+        wfrs = f.get('workflow_run_inputs')
+        has_md5run = False
+        for wfr in wfrs:
+            if wfr.get('workflow', {}).get('title') == 'md5+0.2.6':
+                has_md5run = True
+                break
+        if has_md5run:
+            files.setdefault('files_with_md5run', []).append(f['accession'])
+        else:
+            files.setdefault('files_without_md5run', []).append(f['accession'])
 
-    if files['uploaded_without_md5run'] or files['uploaded_with_md5run']:
+    if files.get('uploaded_without_md5run') or files.get('uploaded_with_md5run'):
         check.status = 'WARN'
         check.summary = 'Some files need md5 run before release'
         check.description = 'Some files with status updloaded or higher are missing md5sum'
@@ -358,6 +365,9 @@ def fastqc_status(connection, **kwargs):
         query += '&lab.display_title=' + lab
     # The search
     res = ff_utils.search_metadata(query, key=my_auth)
+    # check for pre-released status files that have not been uploaded (because they will become restricted)
+    # and remove from res
+    res = [f for f in res if not wfr_utils.prereleased_and_not_uploaded(connection, f)]
     if not res:
         check.summary = 'All Good!'
         return check
@@ -2074,7 +2084,7 @@ def fastq_first_line_status(connection, **kwargs):
     # The search
     print('About to query ES for files')
     res = ff_utils.search_metadata(query, key=my_auth)
-
+    res = [f for f in res if not wfr_utils.prereleased_and_not_uploaded(connection, f)]
     if not res:
         check.summary = "All good!"
         return check

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "foursight"
-version = "4.9.27"
+version = "4.9.28"
 description = "Serverless Chalice Application for Monitoring"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"