diff --git a/usaspending_api/api_contracts/contracts/v2/download/search.md b/usaspending_api/api_contracts/contracts/v2/download/search.md new file mode 100644 index 0000000000..3ca57eaccf --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/download/search.md @@ -0,0 +1,220 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Search Download [/api/v2/download/search/] + +This endpoint is used to search for awards and transactions in the same file. + +## POST + +This route sends a request to the backend to begin generating a zipfile of award data in CSV form for download. + ++ Request (application/json) + + Schema + + { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + } + + + Attributes (object) + + `columns` (optional, array[string]) + + `filters` (required, Filters, fixed-type) + + `file_format` (optional, enum[string]) + The format of the file(s) in the zip file containing the data. + + Default: `csv` + + Members + + `csv` + + `tsv` + + `pstxt` + + `limit` (optional, number) + + `spending_level` (optional, array[enum[string]]) + + Members + + `subawards` + + `transactions` + + `awards` + + Default: [`awards`, `transactions`, `subawards`] + + Body + + { + "filters": { + "agencies": [ + { + "type": "awarding", + "tier": "toptier", + "name": "Department of Agriculture" + } + ], + "keywords": ["Defense"] + }, + "columns": [ + "assistance_award_unique_key", + "assistance_transaction_unique_key", + "award_id_fain", + "award_id_uri", + "modification_number", + "sai_number", + "total_funding_amount" + ], + "spending_level": ["awards","transactions","subawards"] + } + ++ Response 200 (application/json) + + Attributes (object) + + `status_url` (required, string) + The endpoint used to get the status of a download. + + `file_name` (required, string) + Is the name of the zipfile containing CSVs that will be generated (`PrimeAwardsTransactionsAndSubawards` followed by a timestamp). + + `file_url` (required, string) + The URL for the file. + + `download_request` (required, object) + The JSON object used when processing the download. + + Body + + { + "status_url": "http://localhost:8000/api/v2/download/status?file_name=PrimeAwardsTransactionsAndSubawards_2020-01-13_H21M05S48397603.zip", + "file_name": "PrimeAwardsTransactionsAndSubawards_2020-01-13_H21M05S48397603.zip", + "file_url": "/csv_downloads/PrimeAwardsTransactionsAndSubawards_2020-01-13_H21M05S48397603.zip", + "download_request": { + "columns": [ + "assistance_award_unique_key", + "assistance_transaction_unique_key", + "award_id_fain", + "award_id_uri", + "modification_number", + "sai_number", + "total_funding_amount" + ], + "download_types": [ + "elasticsearch_awards", + "elasticsearch_transactions", + "elasticsearch_sub_awards" + ], + "file_format": "csv", + "filters": { + "agencies": [ + { + "type": "awarding", + "tier": "toptier", + "name": "Department of Agriculture" + } + ], + "keywords": ["Defense"] + }, + "limit": 0, + "request_type": "search" + } + } + +# Data Structures + +## Filter Objects + +A more detailed explanation of the available filters can be found [here.](../../../search_filters.md) + +### Filters (object) ++ `agencies` (optional, array[Agency], fixed-type) ++ `award_amounts` (optional, array[AwardAmount], fixed-type) ++ `award_ids` (optional, array[string]) + Award IDs surrounded by double quotes (e.g. `"SPE30018FLJFN"`) will perform exact matches as opposed to the default, fuzzier full text matches. Useful for Award IDs that contain spaces or other word delimiters. ++ `award_type_codes` (optional, array[string]) ++ `contract_pricing_type_codes` (optional, array[string]) ++ `def_codes` (optional, array[DEFC], fixed-type) ++ `extent_competed_type_codes` (optional, array[string]) ++ `federal_account_ids` (optional, array[string]) ++ `keywords` (optional, array[string]) ++ `legal_entities` (optional, array[string]) ++ `naics_codes` (optional, NAICSCodeObject) ++ `object_class_ids` (optional, array[string]) ++ `place_of_performance_locations` (optional, array[Location], fixed-type) ++ `place_of_performance_scope` (optional, string) ++ `program_activity_ids` (optional, array[string]) ++ `program_numbers` (optional, array[string]) ++ `psc_codes` (optional, enum[PSCCodeObject, array[string]]) + Supports new PSCCodeObject or legacy array of codes. ++ `recipient_locations` (optional, array[Location], fixed-type) ++ `recipient_scope` (optional, string) ++ `recipient_search_text` (optional, string) ++ `set_aside_type_codes` (optional, array[string]) ++ `recipient_type_names` (optional, array[string]) ++ `tas_codes` (optional, array[TASCodeObject], fixed-type) ++ `time_period` (optional, array[TimePeriod], fixed-type) ++ `transaction_keyword_search` (optional, string) + Filter awards by keywords in the award's transactions. ++ `treasury_account_components` (optional, array[TreasuryAccountComponentsObject], fixed-type) + +### Agency (object) ++ `name` (required, string) ++ `tier` (required, enum[string]) + + Members + + `toptier` + + `subtier` ++ `type` (required, enum[string]) + + Members + + `funding` + + `awarding` ++ `toptier_name` (optional, string) + Provided when the `name` belongs to a subtier agency + +### AwardAmount (object) ++ `lower_bound` (optional, number) ++ `upper_bound` (optional, number) + +### DEFC (enum[string]) +List of Disaster Emergency Fund (DEF) Codes (DEFC) defined by legislation at the time of writing. +A list of current DEFC can be found [here.](https://files.usaspending.gov/reference_data/def_codes.csv) + + +### Location (object) ++ `country`(required, string) ++ `state` (optional, string) ++ `county` (optional, string) ++ `city` (optional, string) ++ `district_original` (optional, string) + A 2 character code indicating the congressional district + * When provided, a `state` must always be provided as well. + * When provided, a `county` *must never* be provided. + * When provided, `country` must always be "USA". + * When provided, `district_current` *must never* be provided. ++ `district_current` (optional, string) + A 2 character code indicating the current congressional district + * When provided, a `state` must always be provided as well. + * When provided, a `county` *must never* be provided. + * When provided, `country` must always be "USA". + * When provided, `district_original` *must never* be provided. ++ `zip` (optional, string) + +### NAICSCodeObject (object) ++ `require`: [`33`] (optional, array[string], fixed-type) ++ `exclude`: [`3333`] (optional, array[string], fixed-type) + +### PSCCodeObject (object) ++ `require`: [[`Service`, `B`, `B5`]] (optional, array[array[string]], fixed-type) ++ `exclude`: [[`Service`, `B`, `B5`, `B502`]] (optional, array[array[string]], fixed-type) + +### TASCodeObject (object) ++ `require`: [[`091`]] (optional, array[array[string]], fixed-type) ++ `exclude`: [[`091`, `091-0800`]] (optional, array[array[string]], fixed-type) + +### TimePeriod (object) +While accepting the same format, time period filters are interpreted slightly differently between awards, transactions, and subawards. +- [Award Time Period](../../../search_filters.md#award-search-time-period-Object) +- [Transaction Time Period](../../../search_filters.md#transaction-search-time-period-Object) +- [Subawards Time Period](../../../search_filters.md#subaward-search-time-period-Object) + + +### TreasuryAccountComponentsObject (object) ++ `ata` (optional, string, nullable) + Allocation Transfer Agency Identifier - three characters ++ `aid` (required, string) + Agency Identifier - three characters ++ `bpoa` (optional, string, nullable) + Beginning Period of Availability - four digits ++ `epoa` (optional, string, nullable) + Ending Period of Availability - four digits ++ `a` (optional, string, nullable) + Availability Type Code - X or null ++ `main` (required, string) + Main Account Code - four digits ++ `sub` (optional, string, nullable) + Sub-Account Code - three digits diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index fab3340948..5fb400d0d9 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -111,6 +111,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/download/disaster/](/api/v2/download/disaster/)|POST| Returns a zipped file containing Account and Award data for the Disaster Funding | |[/api/v2/download/disaster/recipients/](/api/v2/download/disaster/recipients/)|POST| Returns a zipped file containing Disaster Recipient Funding data | |[/api/v2/download/idv/](/api/v2/download/idv/)|POST| Returns a zipped file containing IDV data | +|[/api/v2/download/search/](/api/v2/download/search/)|POST|Generates zip file for download of award data in CSV format for Awards, Subawards, and Transactions | |[/api/v2/download/status/](/api/v2/download/status/)|GET| gets the current status of a download job that that has been requested with the `v2/download/awards/` or `v2/download/transaction/` endpoint that same day | |[/api/v2/download/transactions/](/api/v2/download/transactions/)|POST|Generates zip file for download of award data in CSV format | |[/api/v2/federal_accounts//](/api/v2/federal_accounts/020-0550/)|GET| Returns a federal account based on its federal account code | diff --git a/usaspending_api/download/download_utils.py b/usaspending_api/download/download_utils.py index a78458e12b..c495acb3f3 100644 --- a/usaspending_api/download/download_utils.py +++ b/usaspending_api/download/download_utils.py @@ -39,6 +39,9 @@ def create_unique_filename(json_request: dict[str, Any], origination: str | None level=level, timestamp=timestamp, ) + elif json_request["request_type"] == "search": + # Search Endpoint uses Award download_types, but has set filename + download_name = f"PrimeAwardsTransactionsAndSubawards_{timestamp}.zip" else: # "award" downloads agency = "" diff --git a/usaspending_api/download/tests/integration/test_download_search.py b/usaspending_api/download/tests/integration/test_download_search.py new file mode 100644 index 0000000000..98d59f0580 --- /dev/null +++ b/usaspending_api/download/tests/integration/test_download_search.py @@ -0,0 +1,243 @@ +import json +from datetime import date +from unittest.mock import Mock + +import pytest +from django.conf import settings +from model_bakery import baker +from rest_framework import status + +from usaspending_api.common.helpers.sql_helpers import get_database_dsn_string +from usaspending_api.download.filestreaming import download_generation +from usaspending_api.download.lookups import JOB_STATUS +from usaspending_api.etl.award_helpers import update_awards +from usaspending_api.search.models import TransactionSearch +from usaspending_api.search.tests.data.utilities import setup_elasticsearch_test + + +@pytest.fixture +def download_test_data(): + # Populate job status lookup table + for js in JOB_STATUS: + baker.make("download.JobStatus", job_status_id=js.id, name=js.name, description=js.desc) + + # Create Awarding Top Agency + ata1 = baker.make( + "references.ToptierAgency", + name="Bureau of Things", + toptier_code="100", + website="http://test.com", + mission="test", + icon_filename="test", + _fill_optional=True, + ) + ata2 = baker.make( + "references.ToptierAgency", + name="Bureau of Stuff", + toptier_code="101", + website="http://test.com", + mission="test", + icon_filename="test", + _fill_optional=True, + ) + + # Create Awarding subs + baker.make("references.SubtierAgency", name="Bureau of Things", _fill_optional=True) + + # Create Awarding Agencies + aa1 = baker.make("references.Agency", id=1, toptier_agency=ata1, toptier_flag=True, _fill_optional=True) + aa2 = baker.make("references.Agency", id=2, toptier_agency=ata2, toptier_flag=True, _fill_optional=True) + + # Create Funding Top Agency + ata3 = baker.make( + "references.ToptierAgency", + name="Bureau of Money", + toptier_code="102", + website="http://test.com", + mission="test", + icon_filename="test", + _fill_optional=True, + ) + + # Create Funding SUB + baker.make("references.SubtierAgency", name="Bureau of Things", _fill_optional=True) + + # Create Funding Agency + baker.make("references.Agency", id=3, toptier_agency=ata3, toptier_flag=True, _fill_optional=True) + + # Create Awards + award1 = baker.make("search.AwardSearch", award_id=123, category="idv", type="A", action_date=date(2026, 3, 3)) + award2 = baker.make( + "search.AwardSearch", award_id=456, category="contracts", type="A", action_date=date(2026, 3, 4) + ) + award3 = baker.make( + "search.AwardSearch", award_id=789, category="assistance", type="A", action_date=date(2026, 3, 5) + ) + + # Create Transactions + baker.make( + TransactionSearch, + transaction_id=1, + award=award1, + action_date="2018-01-01", + type="A", + modification_number=1, + awarding_agency_id=aa1.id, + is_fpds=True, + piid="tc1piid", + awarding_agency_code="100", + awarding_toptier_agency_name="Bureau of Things", + awarding_subtier_agency_name="Bureau of Things", + ) + baker.make( + TransactionSearch, + transaction_id=2, + award=award2, + action_date="2018-01-01", + type="A", + modification_number=1, + awarding_agency_id=aa2.id, + is_fpds=True, + piid="tc2piid", + awarding_agency_code="101", + awarding_toptier_agency_name="Bureau of Stuff", + awarding_subtier_agency_name="Bureau of Things", + ) + baker.make( + TransactionSearch, + transaction_id=3, + award=award3, + action_date="2018-01-01", + type="A", + modification_number=1, + awarding_agency_id=aa2.id, + is_fpds=False, + fain="ta1fain", + awarding_agency_code="101", + awarding_toptier_agency_name="Bureau of Stuff", + awarding_subtier_agency_name="Bureau of Things", + ) + + # Set latest_award for each award + update_awards() + + +@pytest.mark.django_db(databases=[settings.DOWNLOAD_DB_ALIAS, settings.DEFAULT_DB_ALIAS], transaction=True) +def test_download_search_without_columns( + client, monkeypatch, download_test_data, + elasticsearch_award_index, elasticsearch_transaction_index, elasticsearch_subaward_index +): + setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_subaward_index) + download_generation.retrieve_db_string = Mock(return_value=get_database_dsn_string(settings.DOWNLOAD_DB_ALIAS)) + + resp = client.post( + "/api/v2/download/search/", + content_type="application/json", + data=json.dumps({"filters": {"award_type_codes": ["A"]}, "columns": []}), + ) + + assert resp.status_code == status.HTTP_200_OK + assert ".zip" in resp.json()["file_url"] + + +@pytest.mark.django_db(databases=[settings.DOWNLOAD_DB_ALIAS, settings.DEFAULT_DB_ALIAS], transaction=True) +def test_download_search_with_columns( + client, monkeypatch, download_test_data, + elasticsearch_award_index, elasticsearch_transaction_index, elasticsearch_subaward_index +): + """ + Columns that don't exist in a given table just aren't fetched, so they dont error out + behavior regarding potential invalid column headers between awards & transactions should not be necessary + """ + + setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_subaward_index) + download_generation.retrieve_db_string = Mock(return_value=get_database_dsn_string(settings.DOWNLOAD_DB_ALIAS)) + + resp = client.post( + "/api/v2/download/search/", + content_type="application/json", + data=json.dumps( + { + "filters": {"award_type_codes": ["A"]}, + "columns": [ + "total_obligated_amount", + "product_or_service_code", + "product_or_service_code_description", + "naics_code", + "naics_description", + ], + } + ), + ) + + assert resp.status_code == status.HTTP_200_OK + assert ".zip" in resp.json()["file_url"] + + +@pytest.mark.django_db(databases=[settings.DOWNLOAD_DB_ALIAS, settings.DEFAULT_DB_ALIAS], transaction=True) +def test_download_search_bad_filter_type_raises( + client, monkeypatch, download_test_data, elasticsearch_award_index, elasticsearch_transaction_index +): + setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) + download_generation.retrieve_db_string = Mock(return_value=get_database_dsn_string(settings.DOWNLOAD_DB_ALIAS)) + + payload = {"filters": "01", "columns": []} + resp = client.post("/api/v2/download/search/", content_type="application/json", data=json.dumps(payload)) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert resp.json()["detail"] == "Filters parameter not provided as a dict" + + +@pytest.mark.django_db(databases=[settings.DOWNLOAD_DB_ALIAS, settings.DEFAULT_DB_ALIAS], transaction=True) +def test_download_search_with_date_type( + client, monkeypatch, download_test_data, + elasticsearch_award_index, elasticsearch_transaction_index, elasticsearch_subaward_index +): + setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_subaward_index) + download_generation.retrieve_db_string = Mock(return_value=get_database_dsn_string(settings.DOWNLOAD_DB_ALIAS)) + + resp = client.post( + "/api/v2/download/search/", + content_type="application/json", + data=json.dumps( + { + "filters": { + "time_period": [{"date_type": "date_signed", "start_date": "2017-12-31", "end_date": "2018-01-02"}], + } + } + ), + ) + + assert resp.status_code == status.HTTP_200_OK + assert ".zip" in resp.json()["file_url"] + + +@pytest.mark.django_db(databases=[settings.DOWNLOAD_DB_ALIAS, settings.DEFAULT_DB_ALIAS], transaction=True) +def test_download_search_with_invalid_spending_level( + client, monkeypatch, download_test_data, + elasticsearch_award_index, elasticsearch_transaction_index, elasticsearch_subaward_index +): + setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) + setup_elasticsearch_test(monkeypatch, elasticsearch_subaward_index) + download_generation.retrieve_db_string = Mock(return_value=get_database_dsn_string(settings.DOWNLOAD_DB_ALIAS)) + + resp = client.post( + "/api/v2/download/search/", + content_type="application/json", + data=json.dumps( + { + "filters": {"award_type_codes": ["A"]}, + "spending_level": ["sans"] + } + ), + ) + + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert resp.json()["detail"] == 'Invalid parameter: spending_level must be "awards", "subawards", or "transactions"' diff --git a/usaspending_api/download/tests/integration/test_file_generation.py b/usaspending_api/download/tests/integration/test_file_generation.py index a136d921e9..ac3c5128b3 100644 --- a/usaspending_api/download/tests/integration/test_file_generation.py +++ b/usaspending_api/download/tests/integration/test_file_generation.py @@ -67,6 +67,33 @@ def test_get_sub_awards_csv_sources(db): assert csv_sources[1].source_type == "sub_awards" +def test_get_search_csv_sources(db): + # checks to see if search is able to fetch all 3 data types + + original_awards = VALUE_MAPPINGS["elasticsearch_awards"]["filter_function"] + original_transactions = VALUE_MAPPINGS["elasticsearch_transactions"]["filter_function"] + original_subawards = VALUE_MAPPINGS["elasticsearch_sub_awards"]["filter_function"] + + VALUE_MAPPINGS["elasticsearch_awards"]["filter_function"] = MagicMock(returned_value="") + VALUE_MAPPINGS["elasticsearch_transactions"]["filter_function"] = MagicMock(returned_value="") + VALUE_MAPPINGS["elasticsearch_sub_awards"]["filter_function"] = MagicMock(returned_value="") + csv_sources = download_generation.get_download_sources( + { + "download_types": ["elasticsearch_awards", "elasticsearch_transactions", "elasticsearch_sub_awards"], + "filters": {"award_type_codes": list(award_type_mapping.keys())}, + } + ) + assert len(csv_sources) == 6 + VALUE_MAPPINGS["elasticsearch_awards"]["filter_function"] = original_awards + VALUE_MAPPINGS["elasticsearch_transactions"]["filter_function"] = original_transactions + VALUE_MAPPINGS["elasticsearch_sub_awards"]["filter_function"] = original_subawards + assert csv_sources[0].file_type == csv_sources[2].file_type == csv_sources[4].file_type == "d1" + assert csv_sources[1].file_type == csv_sources[1].file_type == csv_sources[1].file_type + assert csv_sources[0].source_type == csv_sources[1].source_type == "elasticsearch_awards" + assert csv_sources[2].source_type == csv_sources[3].source_type == "elasticsearch_transactions" + assert csv_sources[4].source_type == csv_sources[5].source_type == "elasticsearch_sub_awards" + + def test_idv_orders_csv_sources(db): original = VALUE_MAPPINGS["idv_orders"]["filter_function"] VALUE_MAPPINGS["idv_orders"]["filter_function"] = MagicMock(returned_value="") diff --git a/usaspending_api/download/v2/base_download_viewset.py b/usaspending_api/download/v2/base_download_viewset.py index 12bbea94ed..8501872881 100644 --- a/usaspending_api/download/v2/base_download_viewset.py +++ b/usaspending_api/download/v2/base_download_viewset.py @@ -189,11 +189,12 @@ def _get_cached_download( ordered_json_request: str, download_types: Optional[List[str]] = None ) -> Optional[DownloadJob]: # External data types that directly affect download results + external_data_type_name_list = [] + if download_types and "elasticsearch_transactions" in download_types: + external_data_type_name_list.append("es_transactions") if download_types and "elasticsearch_awards" in download_types: - external_data_type_name_list = ["es_awards"] - elif download_types and "elasticsearch_transactions" in download_types: - external_data_type_name_list = ["es_transactions"] - else: + external_data_type_name_list.append("es_awards") + if external_data_type_name_list == []: external_data_type_name_list = [ "fpds", "fabs", diff --git a/usaspending_api/download/v2/request_validations.py b/usaspending_api/download/v2/request_validations.py index 0901f054cc..5c9c28af4f 100644 --- a/usaspending_api/download/v2/request_validations.py +++ b/usaspending_api/download/v2/request_validations.py @@ -1,8 +1,9 @@ import json from copy import deepcopy -from datetime import datetime, MINYEAR, MAXYEAR +from datetime import MAXYEAR, MINYEAR, datetime +from typing import Any, Optional + from django.conf import settings -from typing import Optional from usaspending_api.awards.models import Award from usaspending_api.awards.v2.lookups.lookups import ( @@ -10,9 +11,9 @@ assistance_type_mapping, award_type_mapping, contract_type_mapping, - idv_type_mapping, - grant_type_mapping, direct_payment_type_mapping, + grant_type_mapping, + idv_type_mapping, loan_type_mapping, other_type_mapping, ) @@ -58,18 +59,18 @@ def __init__(self, request_data: dict): self.tinyshield_models = [] - def get_validated_request(self): + def get_validated_request(self) -> dict: models = self.tinyshield_models + self.common_tinyshield_models validated_request = TinyShield(models).block(self._json_request) validated_request["request_type"] = self.name return validated_request - def set_filter_defaults(self, defaults: dict): + def set_filter_defaults(self, defaults: dict) -> None: for key, val in defaults.items(): self._json_request["filters"].setdefault(key, val) @property - def json_request(self): + def json_request(self) -> dict: return deepcopy(self._json_request) @@ -93,7 +94,7 @@ def __init__(self, request_data: dict): else: raise InvalidParameterException('Invalid parameter: constraint_type must be "row_count" or "year"') - def _handle_keyword_search_download(self): + def _handle_keyword_search_download(self) -> None: # Overriding all other filters if the keyword filter is provided in year-constraint download self._json_request["filters"] = {"transaction_keyword_search": self._json_request["filters"]["keywords"]} @@ -115,11 +116,12 @@ def _handle_keyword_search_download(self): }, ] ) + self._json_request = self.get_validated_request() self._json_request["limit"] = settings.MAX_DOWNLOAD_LIMIT self._json_request["filters"]["award_type_codes"] = list(award_type_mapping) - def _handle_custom_award_download(self): + def _handle_custom_award_download(self) -> None: """ Custom Award Download allows different filters than other Award Download Endpoints and thus it needs to be normalized before moving forward @@ -290,10 +292,16 @@ def _handle_custom_award_download(self): "sub_award_types" ] - if "agency" in custom_award_filters: - if "agencies" not in custom_award_filters: - final_award_filters["agencies"] = [] + if "agency" in custom_award_filters or "agencies" in custom_award_filters: + final_award_filters["agencies"] = self._update_custom_award_agencies(custom_award_filters, + filter_all_agencies) + self._json_request["filters"] = final_award_filters + + def _update_custom_award_agencies(self, custom_award_filters: dict, filter_all_agencies: bool) -> list: + agency_output = [] + + if "agency" in custom_award_filters: if filter_all_agencies: toptier_name = "all" else: @@ -307,7 +315,7 @@ def _handle_custom_award_download(self): toptier_name = toptier_name["name"] if "sub_agency" in custom_award_filters: - final_award_filters["agencies"].append( + agency_output.append( { "type": "awarding", "tier": "subtier", @@ -316,16 +324,16 @@ def _handle_custom_award_download(self): } ) else: - final_award_filters["agencies"].append({"type": "awarding", "tier": "toptier", "name": toptier_name}) + agency_output.append({"type": "awarding", "tier": "toptier", "name": toptier_name}) if "agencies" in custom_award_filters: - final_award_filters["agencies"] = [ + agency_output = [ val for val in custom_award_filters["agencies"] if val.get("name", "").lower() != "all" ] - self._json_request["filters"] = final_award_filters + return agency_output - def _handle_advanced_search_download(self): + def _handle_advanced_search_download(self) -> None: self.tinyshield_models.extend( [ *AWARD_FILTER_NO_RECIPIENT_ID, @@ -734,7 +742,70 @@ def __init__(self, request_date: dict): ) -def _validate_award_id(award_id): +class SearchDownloadValidator(DownloadValidatorBase): + name = "search" + + def __init__(self, request_data: dict): + super().__init__(request_data) + self.request_data = request_data + self._json_request["filters"] = _validate_filters_exist(request_data) + self.set_filter_defaults({"award_type_codes": list(award_type_mapping.keys())}) + + self.tinyshield_models.extend( + [ + { + "name": "spending_level", + "key": "spending_level", + "type": "array", + "array_type": "enum", + "enum_values": [ + "awards", + "transactions", + "subawards"], + "optional": True, + "default": ["awards", "transactions", "subawards"], + }, + *AWARD_FILTER_NO_RECIPIENT_ID, + { + "name": "limit", + "key": "limit", + "type": "integer", + "min": 0, + "max": settings.MAX_DOWNLOAD_LIMIT, + "default": settings.MAX_DOWNLOAD_LIMIT, + }, + { + "name": "download_types", + "key": "download_types", + "type": "array", + "array_type": "enum", + "enum_values": [ + "elasticsearch_awards", + "elasticsearch_sub_awards", + "elasticsearch_transactions" + ], + }, + ] + ) + + self._json_request["limit"] = self.request_data.get("limit", settings.MAX_DOWNLOAD_LIMIT) + self._json_request = self.get_validated_request() + + dltypes = [] + for dltype in self.request_data.get("spending_level", ["awards", "transactions", "subawards"]): + if dltype.lower() == "subawards": + dltypes.append("elasticsearch_sub_awards") + elif dltype.lower() in ["awards", "transactions"]: + dltypes.append("elasticsearch_" + dltype) + else: + raise InvalidParameterException( + 'Invalid parameter: spending_level must be "awards", "subawards", or "transactions"' + ) + + self._json_request["download_types"] = dltypes + + +def _validate_award_id(award_id: Any) -> Any: if type(award_id) is int or award_id.isdigit(): filters = {"id": int(award_id)} else: @@ -748,7 +819,7 @@ def _validate_award_id(award_id): return award -def _validate_filters_exist(request_data): +def _validate_filters_exist(request_data: dict) -> dict: filters = request_data.get("filters") if not isinstance(filters, dict): raise InvalidParameterException("Filters parameter not provided as a dict") diff --git a/usaspending_api/download/v2/urls.py b/usaspending_api/download/v2/urls.py index 39cae80f90..104a7363c2 100644 --- a/usaspending_api/download/v2/urls.py +++ b/usaspending_api/download/v2/urls.py @@ -13,6 +13,7 @@ re_path(r"^disaster/recipients", views.DisasterRecipientDownloadViewSet.as_view()), re_path(r"^disaster", views.DisasterDownloadViewSet.as_view()), re_path(r"^idv", views.RowLimitedIDVDownloadViewSet.as_view()), + re_path(r"^search", views.SearchDownloadViewSet.as_view()), re_path(r"^status", DownloadStatusViewSet.as_view()), - re_path(r"^transactions", views.RowLimitedTransactionDownloadViewSet.as_view()), + re_path(r"^transactions", views.RowLimitedTransactionDownloadViewSet.as_view()) ] diff --git a/usaspending_api/download/v2/views.py b/usaspending_api/download/v2/views.py index 9fcac5f00f..a9dee95b21 100644 --- a/usaspending_api/download/v2/views.py +++ b/usaspending_api/download/v2/views.py @@ -1,5 +1,7 @@ import logging +from rest_framework.response import Response + from usaspending_api.download.v2.base_download_viewset import BaseDownloadViewSet from usaspending_api.download.v2.request_validations import ( AccountDownloadValidator, @@ -9,6 +11,7 @@ DisasterDownloadValidator, DisasterRecipientDownloadValidator, IdvDownloadValidator, + SearchDownloadValidator, ) logger = logging.getLogger(__name__) @@ -21,7 +24,7 @@ class RowLimitedAwardDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/awards.md" - def post(self, request): + def post(self, request: dict) -> Response: request.data["award_levels"] = ["elasticsearch_awards", "elasticsearch_sub_awards"] request.data["constraint_type"] = "row_count" return BaseDownloadViewSet.post(self, request, validator_type=AwardDownloadValidator) @@ -34,7 +37,7 @@ class RowLimitedIDVDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/idv.md" - def post(self, request): + def post(self, request: dict) -> Response: request.data["constraint_type"] = "row_count" return BaseDownloadViewSet.post(self, request, validator_type=IdvDownloadValidator) @@ -46,7 +49,7 @@ class RowLimitedContractDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/contract.md" - def post(self, request): + def post(self, request: dict) -> Response: request.data["constraint_type"] = "row_count" return BaseDownloadViewSet.post(self, request, validator_type=ContractDownloadValidator) @@ -58,7 +61,7 @@ class RowLimitedAssistanceDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/assistance.md" - def post(self, request): + def post(self, request: dict) -> Response: request.data["constraint_type"] = "row_count" return BaseDownloadViewSet.post(self, request, validator_type=AssistanceDownloadValidator) @@ -70,7 +73,7 @@ class RowLimitedTransactionDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/transactions.md" - def post(self, request): + def post(self, request: dict) -> Response: request.data["award_levels"] = ["elasticsearch_transactions", "elasticsearch_sub_awards"] request.data["constraint_type"] = "row_count" return BaseDownloadViewSet.post(self, request, validator_type=AwardDownloadValidator) @@ -83,7 +86,7 @@ class AccountDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/accounts.md" - def post(self, request): + def post(self, request: dict) -> Response: """Push a message to SQS with the validated request JSON""" return BaseDownloadViewSet.post(self, request, validator_type=AccountDownloadValidator) @@ -97,7 +100,7 @@ class DisasterDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/disaster.md" - def post(self, request): + def post(self, request: dict) -> Response: """Return url to pre-generated zip file""" return BaseDownloadViewSet.post(self, request, validator_type=DisasterDownloadValidator) @@ -111,5 +114,18 @@ class DisasterRecipientDownloadViewSet(BaseDownloadViewSet): endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/disaster/recipients.md" - def post(self, request): + def post(self, request: dict) -> Response: return BaseDownloadViewSet.post(self, request, validator_type=DisasterRecipientDownloadValidator) + + +class SearchDownloadViewSet(BaseDownloadViewSet): + """ + This route sends a request to begin generating a zip file, + combining award, transaction, and subaward data + """ + + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/download/search.md" + + def post(self, request: dict) -> Response: + request.data["constraint_type"] = "row_count" + return BaseDownloadViewSet.post(self, request, validator_type=SearchDownloadValidator)