fix coderabbit suggestions

Author: willy1989cv

config.ts

@@ -5,19 +5,19 @@ config(); // loads .env
 import { z } from "zod";
 
 const EnvSchema = z.object({
-  SOURCE_CKAN_URL: z.string(),
+  SOURCE_CKAN_URL: z.string().url(),
   SOURCE_CKAN_API_KEY: z.string().optional(),
   SOURCE_CKAN_ORG_ID: z.string().optional(),
 
-  PORTALJS_CKAN_URL: z.string(),
+  PORTALJS_CKAN_URL: z.string().url(),
   PORTALJS_CKAN_API_KEY: z.string().min(1),
   PORTALJS_ORG_ID: z.string().min(1),
 
   CONCURRENCY: z.coerce.number().default(4),
   RATE_LIMIT_RPS: z.coerce.number().default(2),
   RETRY_MAX_ATTEMPTS: z.coerce.number().default(2),
   RETRY_BASE_MS: z.coerce.number().default(500),
-  DRY_RUN: z.preprocess(v => String(v).toLowerCase() === "true", z.boolean()).default(false),
+  DRY_RUN: z.coerce.boolean().default(false),
 
   SINCE_ISO: z.string().optional(),
   STATE_FILE: z.string().default(".harvest_state.json"),

gen-schema.ts

@@ -3,30 +3,34 @@ import fs from "fs";
 import { env } from "./config";
 import { capitalize } from "./src/utils";
 
-
-
 interface SchemaField {
   field_name: string;
   choices?: Array<{ value: string; label: string }>;
-  validators?: string;
+  validators?: string | string[];
 }
 
 async function generateTypes(type: string, schemaUrl: string) {
   const res = await fetch(schemaUrl);
+  if (!res.ok) {
+    const text = await res.text().catch(() => "");
+    throw new Error(
+      `Failed to fetch schema from ${schemaUrl}: ${res.status} ${
+        res.statusText
+      } ${text.slice(0, 300)}`
+    );
+  }
   const data = await res.json();
   const { dataset_fields, resource_fields } = data.result;
 
   const datasetProps = dataset_fields.map((f: SchemaField) => {
     let tsType = "string";
 
     if (f.choices) {
-      tsType = f.choices.map((c) => `"${c.value}"`).join(" | ");
+      tsType = f.choices.map((c) => JSON.stringify(String(c.value))).join(" | ");
     }
 
-    const isRequired =
-      f.validators &&
-      (f.validators.includes("not_empty") ||
-        f.validators.includes("scheming_required"));
+    const validators = Array.isArray(f.validators) ? f.validators : (f.validators ? [f.validators] : []);
+    const isRequired = validators.some(v => v.includes("not_empty") || v.includes("scheming_required"));
 
     return `  ${f.field_name}${isRequired ? "" : "?"}: ${tsType};`;
   });
@@ -46,6 +50,7 @@ ${resourceProps.join("\n")}
 }
 `;
 
+  fs.mkdirSync("schemas", { recursive: true });
   fs.writeFileSync(`schemas/${type}-schema.d.ts`, content);
   console.log(`✅ Types generated in schemas/${type}-schema.d.ts`);
 }

src/map.ts

@@ -11,7 +11,7 @@ export function mapCkanToPortalJS(ds: SourceSchema, owner_org: string): TargetSc
       name: r.name,
       url: r.url,
       format: r.format,
-      id:""
+      ...(r.id ? { id: r.id } : {}),
     })),
 
     language:ds.language || "EN"

src/source.ts

@@ -32,6 +32,7 @@ export async function* iterSourcePackages(pageSize = 25) {
       action,
       {
         ckanUrl: env.SOURCE_CKAN_URL,
+        apiKey: env.SOURCE_CKAN_API_KEY || undefined,
       }
     );