feat(backend): add middleware that handles STREAM KYC/additional data frames (#3706)

* feat(backend): add middleware that handles STREAM KYC/additional data payloads - wip

* feat(backend): add wip comment

* feat: support arbitrary data on prepare packets - WIP

* feat(pay): add controller to handle additional data in STREAM packets

* feat: add STREAM connection id in cache key for KYC response, remove unused env vars

* chore: address PR comments

* chore: add tests for payment decision middleware and interledgerjs changes

* fix: update app data controller ti return F99

* feat: update additional data middleware logic, add pay and stream-receiver updates

* chore(backend): use updated pay and stream-receiver from npm, update lockfile

* feat(backend): refactor partial payment middleware, add handler for partial payments to mock ASE

* fix(backend): fix build errors in CI

* fix(backend): formatting

* fix(backend): formatting

* fix(backend): partial payment decision middleware tests

* fix(backend): formatting

* feat(backend): address review comments

* feat(backend): address PR comments

* fix(backend): fix failing CI tests

* chore: align open-payments-specifications with feature/encrypted-data-exchange

* fix: webhooks tests

* fix: delete flaky test

* fix: backend build

* chore: address review comments

Author: sanducb

localenv/happy-life-bank/docker-compose.yml

@@ -52,6 +52,7 @@ services:
       SIGNATURE_VERSION: 1
       SIGNATURE_SECRET: iyIgCprjb9uL8wFckR+pLEkJWMB7FJhgkvqhTQR/964=
       IDP_SECRET: 2pEcn2kkCclbOHQiGNEwhJ0rucATZhrA807HTm2rNXE=
+      DB_ENCRYPTION_SECRET: 'zO9KogehJECHReHgQr+ZWGkmgOD4AYa4ksUxALSwgM8='
       DISPLAY_NAME: Happy Life Bank
       DISPLAY_ICON: bank-icon.svg
       OPERATOR_TENANT_ID: cf5fd7d3-1eb1-4041-8e43-ba45747e9e5d
@@ -114,6 +115,7 @@ services:
       OPERATOR_TENANT_ID: cf5fd7d3-1eb1-4041-8e43-ba45747e9e5d
       CARD_SERVICE_URL: 'http://happy-life-bank-card-service:4007'
       POS_WEBHOOK_SERVICE_URL: 'http://happy-life-bank-point-of-sale:4008/webhook'
+      DB_ENCRYPTION_SECRET: 'zO9KogehJECHReHgQr+ZWGkmgOD4AYa4ksUxALSwgM8='
       WALLET_ADDRESS_NOT_FOUND_POLLING_ENABLED: true
     depends_on:
       - cloud-nine-backend

localenv/mock-account-servicing-entity/app/lib/webhooks.server.ts

@@ -186,6 +186,62 @@ export async function handleIncomingPaymentCompletedExpired(
   return
 }
 
+export async function handleIncomingPartialPaymentReceived(
+  wh: Webhook,
+  options?: TenantOptions
+) {
+  if (wh.type !== WebhookEventType.IncomingPaymentPartialPaymentReceived) {
+    throw new Error(
+      'Invalid event type when handling incoming partial payment webhook'
+    )
+  }
+
+  const incomingPaymentId = wh.data['id'] as string | undefined
+  if (!incomingPaymentId) {
+    throw new Error('No incomingPaymentId found on webhook data')
+  }
+
+  const partialIncomingPaymentId = wh.data['partialIncomingPaymentId'] as
+    | string
+    | undefined
+  if (!partialIncomingPaymentId) {
+    throw new Error('No partialIncomingPaymentId found on webhook data')
+  }
+
+  const rawDataToTransmit = wh.data['dataToTransmit'] as string | undefined
+  if (!rawDataToTransmit) {
+    throw new Error('No dataToTransmit found on webhook data')
+  }
+
+  await generateApolloClient(options)
+    .mutate({
+      mutation: gql`
+        mutation ConfirmPartialIncomingPayment(
+          $input: ConfirmPartialIncomingPaymentInput!
+        ) {
+          confirmPartialIncomingPayment(input: $input) {
+            success
+          }
+        }
+      `,
+      variables: {
+        input: {
+          incomingPaymentId,
+          partialIncomingPaymentId
+        }
+      }
+    })
+    .then((query): LiquidityMutationResponse => {
+      if (query.data) {
+        return query.data.confirmPartialIncomingPayment
+      } else {
+        throw new Error('Data was empty')
+      }
+    })
+
+  return
+}
+
 export async function handleWalletAddressWebMonetization(
   wh: Webhook,
   options?: TenantOptions

localenv/mock-account-servicing-entity/app/routes/webhooks.ts

@@ -3,12 +3,11 @@ import { json } from '@remix-run/node'
 import {
   handleLowLiquidity,
   handleWalletAddressNotFound,
-  handleWalletAddressWebMonetization
-} from '~/lib/webhooks.server'
-import {
+  handleWalletAddressWebMonetization,
   handleOutgoingPaymentCreated,
   handleOutgoingPaymentCompletedFailed,
-  handleIncomingPaymentCompletedExpired
+  handleIncomingPaymentCompletedExpired,
+  handleIncomingPartialPaymentReceived
 } from '~/lib/webhooks.server'
 import { WebhookEventType, Webhook } from 'mock-account-service-lib'
 import { getTenantCredentials } from '~/lib/utils'
@@ -36,6 +35,9 @@ export async function action({ request }: ActionFunctionArgs) {
         break
       case WebhookEventType.IncomingPaymentCreated:
         break
+      case WebhookEventType.IncomingPaymentPartialPaymentReceived:
+        await handleIncomingPartialPaymentReceived(wh, tenantOptions)
+        break
       case WebhookEventType.IncomingPaymentCompleted:
       case WebhookEventType.IncomingPaymentExpired:
         await handleIncomingPaymentCompletedExpired(wh, tenantOptions)

packages/backend/package.json

@@ -32,7 +32,7 @@
     "@types/tmp": "^0.2.6",
     "@types/uuid": "^9.0.8",
     "cross-fetch": "^4.1.0",
-    "ilp-protocol-stream": "^2.7.2-alpha.2",
+    "ilp-protocol-stream": "2.7.2-alpha.3",
     "jest-environment-node": "^29.7.0",
     "jest-openapi": "^0.14.2",
     "nock": "14.0.0-beta.19",
@@ -57,8 +57,8 @@
     "@interledger/http-signature-utils": "2.0.2",
     "@interledger/open-payments": "7.3.0",
     "@interledger/openapi": "2.0.2",
-    "@interledger/pay": "0.4.0-alpha.9",
-    "@interledger/stream-receiver": "^0.3.3-alpha.3",
+    "@interledger/pay": "0.4.0-alpha.10",
+    "@interledger/stream-receiver": "0.3.3-alpha.4",
     "@koa/cors": "^5.0.0",
     "@koa/router": "^12.0.2",
     "@opentelemetry/api": "^1.8.0",

packages/backend/src/config/app.ts

@@ -212,6 +212,18 @@ export const Config = {
   ),
   /** Optional base64-encoded key for encrypting partial-payment payload fields in webhooks. */
   dbEncryptionSecret: process.env.DB_ENCRYPTION_SECRET,
+  enablePartialPaymentDecision: envBool(
+    'ENABLE_PARTIAL_PAYMENT_DECISION',
+    false
+  ),
+  partialPaymentDecisionMaxWaitMs: envInt(
+    'PARTIAL_PAYMENT_DECISION_MAX_WAIT_MS',
+    1500
+  ),
+  partialPaymentDecisionSafetyMarginMs: envInt(
+    'PARTIAL_PAYMENT_DECISION_SAFETY_MARGIN_MS',
+    100
+  ),
   cardServiceUrl: optional(envString, 'CARD_SERVICE_URL'),
   posServiceUrl: optional(envString, 'POS_SERVICE_URL'),
   posWebhookServiceUrl: optional(envString, 'POS_WEBHOOK_SERVICE_URL'),

packages/backend/src/index.ts

@@ -377,13 +377,13 @@ export function initIocContainer(
 
   container.singleton('incomingPaymentService', async (deps) => {
     return await createIncomingPaymentService({
-      redis: await deps.use('redis'),
       logger: await deps.use('logger'),
       knex: await deps.use('knex'),
       accountingService: await deps.use('accountingService'),
       walletAddressService: await deps.use('walletAddressService'),
       assetService: await deps.use('assetService'),
-      config: await deps.use('config')
+      config: await deps.use('config'),
+      redis: await deps.use('redis')
     })
   })
   container.singleton('remoteIncomingPaymentService', async (deps) => {