Skip to content

Commit 954df5b

Browse files
levinmrlevinmr
authored
Merge pull request #807 from 18F/staging
Production deploy to fix sampling in 30 day hostname report
2 parents 87266ed + 57689ef commit 954df5b

File tree

85 files changed

+8008
-4600
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+8008
-4600
lines changed

.circleci/config.yml renamed to .circleci/config.yml.old

File renamed without changes.

‎.github/workflows/ci.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
on:
2+
push:
3+
pull_request:
4+
5+
jobs:
6+
lint:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- name: Code checkout
10+
uses: actions/checkout@v4
11+
- name: Install node
12+
uses: actions/setup-node@v4
13+
with:
14+
node-version: "lts/*"
15+
cache: 'npm'
16+
- name: Install node dependencies
17+
run: npm ci
18+
- name: Lint javascript
19+
run: npm run lint
20+
test:
21+
needs: lint
22+
runs-on: ubuntu-latest
23+
# Start Postgres as a service, wait until healthy. Uses latest Postgres version.
24+
services:
25+
postgres:
26+
image: postgres:latest
27+
env:
28+
POSTGRES_DB: analytics_reporter_test
29+
POSTGRES_USER: analytics
30+
POSTGRES_PASSWORD: 123abc
31+
ports:
32+
- 5432:5432
33+
options:
34+
--health-cmd pg_isready
35+
--health-interval 10s
36+
--health-timeout 5s
37+
--health-retries 5
38+
steps:
39+
- name: Code checkout
40+
uses: actions/checkout@v4
41+
- name: Install node
42+
uses: actions/setup-node@v4
43+
with:
44+
node-version: "lts/*"
45+
cache: 'npm'
46+
- name: Install node dependencies
47+
run: npm ci
48+
- name: Run tests
49+
run: npm test
50+
deploy_dev:
51+
needs:
52+
- lint
53+
- test
54+
if: github.ref == 'refs/heads/develop'
55+
uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
56+
with:
57+
ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
58+
ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
59+
APP_NAME: ${{ vars.APP_NAME_DEV }}
60+
CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
61+
CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_DEV }}
62+
DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_DEV }}
63+
NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_DEV }}
64+
S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_DEV }}
65+
secrets:
66+
ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
67+
CF_USERNAME: ${{ secrets.CF_USERNAME_DEV }}
68+
CF_PASSWORD: ${{ secrets.CF_PASSWORD_DEV }}
69+
GA4_CREDS: ${{ secrets.GA4_CREDS }}
70+
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_DEV }}
71+
deploy_stg:
72+
needs:
73+
- lint
74+
- test
75+
if: github.ref == 'refs/heads/staging'
76+
uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
77+
with:
78+
ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
79+
ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
80+
APP_NAME: ${{ vars.APP_NAME_STG }}
81+
CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
82+
CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_STG }}
83+
DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_STG }}
84+
NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_STG }}
85+
S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_STG }}
86+
secrets:
87+
ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
88+
CF_USERNAME: ${{ secrets.CF_USERNAME_STG }}
89+
CF_PASSWORD: ${{ secrets.CF_PASSWORD_STG }}
90+
GA4_CREDS: ${{ secrets.GA4_CREDS }}
91+
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_STG }}
92+
deploy_prd:
93+
needs:
94+
- lint
95+
- test
96+
if: github.ref == 'refs/heads/master'
97+
uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
98+
with:
99+
ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
100+
ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
101+
APP_NAME: ${{ vars.APP_NAME_PRD }}
102+
CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
103+
CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_PRD }}
104+
DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_PRD }}
105+
NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_PRD }}
106+
S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_PRD }}
107+
secrets:
108+
ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
109+
CF_USERNAME: ${{ secrets.CF_USERNAME_PRD }}
110+
CF_PASSWORD: ${{ secrets.CF_PASSWORD_PRD }}
111+
GA4_CREDS: ${{ secrets.GA4_CREDS }}
112+
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_PRD }}

‎.github/workflows/deploy.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
ANALYTICS_KEY_FILE_NAME:
5+
required: true
6+
type: string
7+
ANALYTICS_REPORT_EMAIL:
8+
required: true
9+
type: string
10+
APP_NAME:
11+
required: true
12+
type: string
13+
CF_ORGANIZATION_NAME:
14+
required: true
15+
type: string
16+
CF_SPACE_NAME:
17+
required: true
18+
type: string
19+
DB_SERVICE_NAME:
20+
required: true
21+
type: string
22+
NEW_RELIC_APP_NAME:
23+
type: string
24+
S3_SERVICE_NAME:
25+
required: true
26+
type: string
27+
secrets:
28+
ANALYTICS_CREDENTIALS:
29+
required: true
30+
CF_USERNAME:
31+
required: true
32+
CF_PASSWORD:
33+
required: true
34+
GA4_CREDS:
35+
required: true
36+
NEW_RELIC_LICENSE_KEY:
37+
38+
env:
39+
ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
40+
ANALYTICS_KEY_FILE_NAME: ${{ inputs.ANALYTICS_KEY_FILE_NAME }}
41+
ANALYTICS_REPORT_EMAIL: ${{ inputs.ANALYTICS_REPORT_EMAIL }}
42+
APP_NAME: ${{ inputs.APP_NAME }}
43+
CF_USERNAME: ${{ secrets.CF_USERNAME }}
44+
CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
45+
CF_ORGANIZATION_NAME: ${{ inputs.CF_ORGANIZATION_NAME }}
46+
CF_SPACE_NAME: ${{ inputs.CF_SPACE_NAME }}
47+
DB_SERVICE_NAME: ${{ inputs.DB_SERVICE_NAME }}
48+
GA4_CREDS: ${{ secrets.GA4_CREDS }}
49+
NEW_RELIC_APP_NAME: ${{ inputs.NEW_RELIC_APP_NAME }}
50+
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
51+
S3_SERVICE_NAME: ${{ inputs.S3_SERVICE_NAME }}
52+
53+
jobs:
54+
deploy_reporter:
55+
runs-on: ubuntu-latest
56+
steps:
57+
- name: Code checkout
58+
uses: actions/checkout@v4
59+
- name: Install node
60+
uses: actions/setup-node@v4
61+
with:
62+
node-version: "lts/*"
63+
cache: 'npm'
64+
- name: Install node dependencies
65+
run: npm ci
66+
- name: Install cloud foundry CLI for interacting with cloud.gov
67+
run: |
68+
sudo curl -v -L -o cf8-cli-installer_8.7.4_x86-64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=8.7.4'
69+
sudo dpkg -i cf8-cli-installer_8.7.4_x86-64.deb
70+
- name: Write Google GA4 Credentials file from the value in GA4_CREDS env var.
71+
run: |
72+
echo $GA4_CREDS > ./my-analytics-ga4-65057af58daa.json
73+
- name: Run envsubst on manifest.yml to set environment specific values
74+
run: |
75+
mv manifest.yml manifest.yml.src
76+
envsubst < manifest.yml.src > manifest.yml
77+
cat manifest.yml
78+
- name: Replace config.js and knexfile.js with .cloudgov versions of those files
79+
run: |
80+
rm ./src/config.js
81+
mv ./src/config.cloudgov.js ./src/config.js
82+
rm knexfile.js
83+
mv knexfile.cloudgov.js knexfile.js
84+
- name: Login to cloud.gov and deploy
85+
run: |
86+
set -e
87+
# Log into cloud.gov
88+
cf api api.fr.cloud.gov
89+
cf login -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORGANIZATION_NAME -s $CF_SPACE_NAME
90+
cf push -f "./manifest.yml" --strategy rolling
91+
cf logout

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
npm-debug.log
1212
*.swp
1313
.git
14+
/.nyc_output
1415

1516
# Track selected JSON files
1617
!package.json

‎README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,21 @@ Compose:
418418
docker-compose up
419419
```
420420

421+
## Linting
422+
423+
This repo uses Eslint and Prettier for code static analysis and formatting. Run
424+
the linter with:
425+
426+
```shell
427+
npm run lint
428+
```
429+
430+
Automatically fix lint issues with:
431+
432+
```shell
433+
npm run lint:fix
434+
```
435+
421436
## Running the unit tests
422437

423438
The unit tests for this repo require a local PostgreSQL database. You can run a
@@ -441,6 +456,16 @@ Run the tests (pre-test hook runs DB migrations):
441456
npm test
442457
```
443458

459+
### Running the unit tests with code coverage reporting
460+
461+
If you wish to see a code coverage report after running the tests, use the
462+
following command. This runs the DB migrations, tests, and the NYC code coverage
463+
tool:
464+
465+
```shell
466+
npm run coverage
467+
```
468+
444469
## Public domain
445470

446471
This project is in the worldwide [public domain](LICENSE.md). As stated in [CONTRIBUTING](CONTRIBUTING.md):

‎deploy/cron.js‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ if (process.env.NEW_RELIC_APP_NAME) {
77
}
88

99
const spawn = require("child_process").spawn;
10-
const logger = require('../src/logger').initialize();
10+
const logger = require("../src/logger").initialize();
1111

1212
logger.info("===================================");
1313
logger.info("=== STARTING ANALYTICS-REPORTER ===");
@@ -25,41 +25,41 @@ const runScriptWithLogName = (scriptPath, scriptLoggingName) => {
2525
childProcess.stdout.on("data", (data) => {
2626
logger.info(`[${scriptLoggingName}]`);
2727
// Writes logging output from child processes to console.
28-
console.log(data.toString().trim())
28+
console.log(data.toString().trim());
2929
});
3030

3131
childProcess.stderr.on("data", (data) => {
3232
logger.error(`[${scriptLoggingName}]`);
3333
// Writes error logging output from child processes to console.
34-
console.log(data.toString().trim())
34+
console.log(data.toString().trim());
3535
});
3636

37-
childProcess.on("exit", (code, signal) => {
37+
childProcess.on("close", (code, signal) => {
3838
logger.info(`${scriptLoggingName} exitted with code: ${code}`);
3939
if (signal) {
4040
logger.info(`${scriptLoggingName} received signal: ${signal}`);
4141
}
4242
});
43-
}
43+
};
4444

4545
const api_ua_run = () => {
46-
runScriptWithLogName(`${scriptUARootPath}/api.sh`, 'ua - api.sh')
46+
runScriptWithLogName(`${scriptUARootPath}/api.sh`, "ua - api.sh");
4747
};
4848

4949
const api_run = () => {
50-
runScriptWithLogName(`${scriptRootPath}/api.sh`, 'api.sh')
50+
runScriptWithLogName(`${scriptRootPath}/api.sh`, "api.sh");
5151
};
5252

5353
const daily_run = () => {
54-
runScriptWithLogName(`${scriptRootPath}/daily.sh`, 'daily.sh')
54+
runScriptWithLogName(`${scriptRootPath}/daily.sh`, "daily.sh");
5555
};
5656

5757
const hourly_run = () => {
58-
runScriptWithLogName(`${scriptRootPath}/hourly.sh`, 'hourly.sh')
58+
runScriptWithLogName(`${scriptRootPath}/hourly.sh`, "hourly.sh");
5959
};
6060

6161
const realtime_run = () => {
62-
runScriptWithLogName(`${scriptRootPath}/realtime.sh`, 'realtime.sh')
62+
runScriptWithLogName(`${scriptRootPath}/realtime.sh`, "realtime.sh");
6363
};
6464

6565
/**
@@ -72,7 +72,7 @@ const calculateNextDailyRunTimeOffset = () => {
7272
currentTime.getFullYear(),
7373
currentTime.getMonth(),
7474
currentTime.getDate() + 1,
75-
10 - currentTime.getTimezoneOffset() / 60
75+
10 - currentTime.getTimezoneOffset() / 60,
7676
);
7777
return (nextRunTime - currentTime) % (1000 * 60 * 60 * 24);
7878
};
@@ -92,14 +92,15 @@ setTimeout(() => {
9292
// Run at 10 AM UTC, then every 24 hours afterwards
9393
daily_run();
9494
setInterval(daily_run, 1000 * 60 * 60 * 24);
95-
//api
95+
// API
9696
api_run();
9797
setInterval(api_run, 1000 * 60 * 60 * 24);
98-
//ua api
98+
// UA API
9999
api_ua_run();
100100
setInterval(api_ua_run, 1000 * 60 * 60 * 24);
101101
}, calculateNextDailyRunTimeOffset());
102-
//hourly
102+
// hourly
103103
setInterval(hourly_run, 1000 * 60 * 60);
104-
//realtime
105-
setInterval(realtime_run, 1000 * 60 * 5);
104+
// realtime. Runs every 15 minutes.
105+
// Google updates realtime reports every 30 minutes, so there is some overlap.
106+
setInterval(realtime_run, 1000 * 60 * 15);

‎eslint.config.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
const { configs: eslintConfigs } = require("@eslint/js");
2+
const eslintPluginPrettierRecommended = require("eslint-plugin-prettier/recommended");
3+
const globals = require("globals");
4+
5+
module.exports = [
6+
{
7+
languageOptions: {
8+
globals: {
9+
...globals.node,
10+
...globals.mocha,
11+
},
12+
},
13+
},
14+
{
15+
// UA code is deprecated, so don't bother with static analysis rules.
16+
ignores: ["ua/**/*.js"],
17+
...eslintConfigs.recommended,
18+
},
19+
eslintPluginPrettierRecommended,
20+
];

0 commit comments

Comments
 (0)