Update dependencies and change parquet engine in tests (#211)

Author: andersy005

offsets_db_api/tasks.py

@@ -180,7 +180,11 @@ def process_dataframe(
             if 'ARRAY' in str(dtype) and col_name in df.columns:
                 logger.info(f'Converting column {col_name} to PostgreSQL array format')
                 df[col_name] = df[col_name].apply(
-                    lambda x: '{' + ','.join(str(i) for i in x) + '}' if isinstance(x, list) else x
+                    lambda x: (
+                        '{' + ','.join(str(i) for i in x) + '}'
+                        if (hasattr(x, '__iter__') and not isinstance(x, str))
+                        else x
+                    )
                 )
 
     with engine.begin() as conn:
@@ -404,7 +408,7 @@ async def process_files(*, engine, session, files: list[File], chunk_size: int =
             if file.category == 'credits':
                 logger.info(f'📚 Loading credit file: {file.url}')
                 data = (
-                    pd.read_parquet(file.url, engine='fastparquet')
+                    pd.read_parquet(file.url, engine='pyarrow')
                     .reset_index(drop=True)
                     .reset_index()
                     .rename(columns={'index': 'id'})
@@ -428,7 +432,7 @@ async def process_files(*, engine, session, files: list[File], chunk_size: int =
 
             elif file.category == 'projects':
                 logger.info(f'📚 Loading project file: {file.url}')
-                data = pd.read_parquet(file.url, engine='fastparquet')
+                data = pd.read_parquet(file.url, engine='pyarrow')
                 df = project_schema.validate(data)
                 project_dtype_dict = {
                     'project_id': String,
@@ -482,7 +486,7 @@ async def process_files(*, engine, session, files: list[File], chunk_size: int =
         metrics['file_metrics'][file.id]['category'] = file.category
         try:
             logger.info(f'📚 Loading clip file: {file.url}')
-            data = pd.read_parquet(file.url, engine='fastparquet')
+            data = pd.read_parquet(file.url, engine='pyarrow')
             clips_dfs.append(data)
             update_file_status(file, session, 'success')
             metrics['file_metrics'][file.id]['status'] = 'success'