Skip to content

Commit 9ce48d7

Browse files
andybywireandybywire
authored
Merge pull request #41 from andybywire/kg
Simplify and remove server-side fallback data
2 parents 0719899 + 9d5a079 commit 9ce48d7

File tree

3 files changed

+59
-180
lines changed

3 files changed

+59
-180
lines changed

‎.github/workflows/build-prod.yml‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,6 @@ jobs:
5959
env:
6060
NODE_ENV: production
6161
KG_AUTH: ${{ secrets.KG_AUTH }}
62-
continue-on-error: false
63-
64-
- name: Check for KG fallback usage
65-
run: |
66-
if grep -q "Using fallback data" web/_site/index.html; then
67-
echo "WARNING: Using fallback KG data" >> $GITHUB_STEP_SUMMARY
68-
echo "::warning::Using fallback KG data"
69-
fi
7062

7163
- name: Verify build
7264
run: ls -la ./_site
@@ -79,22 +71,6 @@ jobs:
7971
sudo chmod 600 ~/.ssh/do_uxm_prod
8072
ssh-keyscan -H "24.199.98.130" > ~/.ssh/known_hosts
8173
82-
- name: Manage fallback data directory
83-
run: |
84-
# Create directory if it doesn't exist
85-
ssh -i ~/.ssh/do_uxm_prod prod@24.199.98.130 "mkdir -p /var/www/uxm/fallback-data"
86-
87-
# Keep only the 2 most recent backup files
88-
ssh -i ~/.ssh/do_uxm_prod prod@24.199.98.130 "ls -t /var/www/uxm/fallback-data/kg-data-*.json | tail -n +3 | xargs -r rm"
89-
90-
- name: Upload fallback data
91-
run: |
92-
if [ -f "web/_data/fallback-data/kg-data.json" ]; then
93-
# Add timestamp to filename to keep history
94-
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
95-
scp -i ~/.ssh/do_uxm_prod web/_data/fallback-data/kg-data.json prod@24.199.98.130:/var/www/uxm/fallback-data/kg-data-${TIMESTAMP}.json
96-
fi
97-
9874
- name: Clean & FTP Upload
9975
run: |
10076
ssh -i ~/.ssh/do_uxm_prod prod@24.199.98.130 "rm -rf /var/www/uxm/html/*"

‎web/.eleventy.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ export default function (eleventyConfig) {
3030

3131
// Watch all asset directories for changes
3232
eleventyConfig.addWatchTarget('_src/**/*') // Watch everything in _src
33-
eleventyConfig.watchIgnores.add('_data/fallback-data/**/*') // Ignore fallback data directory
3433

3534
eleventyConfig.setServerOptions({
3635
showAllHosts: true,

‎web/_data/methods.js‎

Lines changed: 59 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -1,71 +1,16 @@
11
import {client} from '../utils/sanityClient.js'
22
import {toHTML} from '@portabletext/to-html'
33
import groq from 'groq'
4-
import {readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync, unlinkSync} from 'fs'
5-
import {join} from 'path'
6-
import * as dotenv from 'dotenv'
7-
// import { uxmComponents } from '../utils/serializers.js'
4+
import dotenv from 'dotenv'
85

96
// Load environment variables from .env.local in development only
107
if (process.env.NODE_ENV !== 'production') {
118
dotenv.config({ path: '.env.local' })
129
}
1310

14-
// Constants for fallback data
15-
const FALLBACK_DIR = process.env.NODE_ENV === 'production'
16-
? '/var/www/uxm/fallback-data'
17-
: join(process.cwd(), '_data/fallback-data')
18-
const FALLBACK_FILE = join(FALLBACK_DIR, 'kg-data.json')
19-
20-
// Clean up old fallback files in development
21-
function cleanupFallbackData() {
22-
if (process.env.NODE_ENV !== 'production') {
23-
try {
24-
if (existsSync(FALLBACK_DIR)) {
25-
// Get all fallback files sorted by modification time
26-
const files = readdirSync(FALLBACK_DIR)
27-
.filter(file => file.startsWith('kg-data-') && file.endsWith('.json'))
28-
.map(file => ({
29-
name: file,
30-
path: join(FALLBACK_DIR, file),
31-
time: statSync(join(FALLBACK_DIR, file)).mtime.getTime()
32-
}))
33-
.sort((a, b) => b.time - a.time)
34-
35-
// Keep only the 2 most recent files
36-
files.slice(2).forEach(file => {
37-
unlinkSync(file.path)
38-
console.log(`Removed old fallback file: ${file.name}`)
39-
})
40-
}
41-
} catch (error) {
42-
console.warn('Error cleaning up fallback data:', error.message)
43-
}
44-
}
45-
}
46-
47-
// Get the most recent fallback file
48-
function getMostRecentFallback() {
49-
try {
50-
if (existsSync(FALLBACK_DIR)) {
51-
const files = readdirSync(FALLBACK_DIR)
52-
.filter(file => file.startsWith('kg-data-') && file.endsWith('.json'))
53-
.map(file => ({
54-
name: file,
55-
path: join(FALLBACK_DIR, file),
56-
time: statSync(join(FALLBACK_DIR, file)).mtime.getTime()
57-
}))
58-
.sort((a, b) => b.time - a.time)
59-
60-
if (files.length > 0) {
61-
return files[0].path
62-
}
63-
}
64-
return null
65-
} catch (error) {
66-
console.warn('Error finding most recent fallback:', error.message)
67-
return null
68-
}
11+
// Ensure required environment variables are set
12+
if (!process.env.KG_AUTH) {
13+
throw new Error('KG_AUTH environment variable is required')
6914
}
7015

7116
// Transform SPARQL results into a more usable format
@@ -78,77 +23,36 @@ function transformKgData(kgData) {
7823
}))
7924
}
8025

81-
// Get shared output from KG with fallback mechanism
26+
// Get shared output from KG
8227
async function getSharedOutput() {
83-
try {
84-
const kgData = await fetch('http://kg.uxmethods.org/repositories/uxm', {
85-
method: 'POST',
86-
headers: {
87-
'Content-Type': 'application/sparql-query',
88-
Accept: 'application/sparql-results+json',
89-
Authorization: 'Basic ' + btoa(process.env.KG_AUTH),
90-
},
91-
body: `
92-
PREFIX : <https://uxmethods.org/>
93-
PREFIX uxmo: <https://uxmethods.org/ontology/>
94-
95-
SELECT ?origin ?destination (COUNT(?output) AS ?sharedOutputCount)
96-
(GROUP_CONCAT(DISTINCT ?output; SEPARATOR=",") AS ?sharedOutput)
97-
WHERE {
98-
?origin uxmo:hasOutput ?output.
99-
?destination uxmo:hasInput ?output.
100-
}
101-
GROUP BY ?origin ?destination
102-
ORDER BY DESC(?sharedOutputCount)
103-
`,
104-
})
105-
106-
if (!kgData.ok) {
107-
throw new Error(`HTTP error! status: ${kgData.status}`)
108-
}
109-
110-
const data = await kgData.json()
111-
const transformedData = transformKgData(data)
112-
113-
// Store successful response as backup
114-
const backupData = {
115-
timestamp: new Date().toISOString(),
116-
data: transformedData
117-
}
118-
119-
// Only create directory and write files in development
120-
if (process.env.NODE_ENV !== 'production') {
121-
// Ensure fallback directory exists
122-
if (!existsSync(FALLBACK_DIR)) {
123-
mkdirSync(FALLBACK_DIR, { recursive: true })
124-
}
125-
126-
const timestamp = new Date().toISOString().replace(/[:.]/g, '-')
127-
const backupFile = join(FALLBACK_DIR, `kg-data-${timestamp}.json`)
128-
writeFileSync(backupFile, JSON.stringify(backupData, null, 2))
129-
cleanupFallbackData()
130-
}
131-
132-
return transformedData
133-
} catch (error) {
134-
console.warn('Failed to fetch from KG, attempting to use fallback data:', error.message)
135-
136-
try {
137-
// Try to read fallback data
138-
const fallbackPath = process.env.NODE_ENV === 'production'
139-
? FALLBACK_FILE
140-
: getMostRecentFallback()
28+
const kgData = await fetch('http://kg.uxmethods.org/repositories/uxm', {
29+
method: 'POST',
30+
headers: {
31+
'Content-Type': 'application/sparql-query',
32+
Accept: 'application/sparql-results+json',
33+
Authorization: 'Basic ' + btoa(process.env.KG_AUTH),
34+
},
35+
body: `
36+
PREFIX : <https://uxmethods.org/>
37+
PREFIX uxmo: <https://uxmethods.org/ontology/>
38+
39+
SELECT ?origin ?destination (COUNT(?output) AS ?sharedOutputCount)
40+
(GROUP_CONCAT(DISTINCT ?output; SEPARATOR=",") AS ?sharedOutput)
41+
WHERE {
42+
?origin uxmo:hasOutput ?output.
43+
?destination uxmo:hasInput ?output.
44+
}
45+
GROUP BY ?origin ?destination
46+
ORDER BY DESC(?sharedOutputCount)
47+
`,
48+
})
14149

142-
if (fallbackPath && existsSync(fallbackPath)) {
143-
const fallbackData = JSON.parse(readFileSync(fallbackPath, 'utf-8'))
144-
console.warn('Using fallback data from:', fallbackData.timestamp)
145-
return fallbackData.data
146-
}
147-
throw new Error('No fallback data available')
148-
} catch (fallbackError) {
149-
throw new Error(`Both KG fetch and fallback failed: ${fallbackError.message}`)
150-
}
50+
if (!kgData.ok) {
51+
throw new Error(`Failed to fetch from KG: HTTP error! status: ${kgData.status}`)
15152
}
53+
54+
const data = await kgData.json()
55+
return transformKgData(data)
15256
}
15357

15458
function prepareMethod(methods, methodPreviews, kgData) {
@@ -186,36 +90,36 @@ async function getMethods() {
18690
const [methods, kgData] = await Promise.all([
18791
client.fetch(groq`
18892
*[_type == "method"] | order(title) {
189-
title,
190-
"type": "method",
191-
"slug": slug.current,
192-
"uri": uri.current,
193-
"createdAt": dateStamp.createdAt,
194-
"revisedAt": dateStamp.revisedAt,
195-
metaDescription,
196-
"heroImage": {
197-
"credit": heroImage.asset->creditLine,
198-
"source": heroImage.asset->source.url,
199-
"url": heroImage.asset->url,
200-
...heroImage
201-
},
202-
overview,
203-
steps,
204-
stepSources,
205-
dateStamps,
206-
"outcomes": output[]->{
207-
prefLabel,
208-
definition,
209-
},
210-
"resources": *[_type == "resource" && references(^._id)]{
21193
title,
212-
author,
213-
resourceUrl,
214-
resourceImage,
215-
"publisher": publisher.pubName
94+
"type": "method",
95+
"slug": slug.current,
96+
"uri": uri.current,
97+
"createdAt": dateStamp.createdAt,
98+
"revisedAt": dateStamp.revisedAt,
99+
metaDescription,
100+
"heroImage": {
101+
"credit": heroImage.asset->creditLine,
102+
"source": heroImage.asset->source.url,
103+
"url": heroImage.asset->url,
104+
...heroImage
105+
},
106+
overview,
107+
steps,
108+
stepSources,
109+
dateStamps,
110+
"outcomes": output[]->{
111+
prefLabel,
112+
definition,
113+
},
114+
"resources": *[_type == "resource" && references(^._id)]{
115+
title,
116+
author,
117+
resourceUrl,
118+
resourceImage,
119+
"publisher": publisher.pubName
120+
}
216121
}
217-
}
218-
`),
122+
`),
219123
getSharedOutput()
220124
])
221125

0 commit comments

Comments
 (0)