From 007e77703f4a88ca4346ac48d5dd7c1298bfd151 Mon Sep 17 00:00:00 2001
From: PhuongTa <duy.ta@materna.group>
Date: Mon, 16 Sep 2024 17:20:01 +0200
Subject: [PATCH] add federated catalog extension

---
 Readme.md                                     | 13 ++-
 docker-compose.yaml                           | 12 +++
 sln-connector/gradle/libs.versions.toml       |  6 ++
 .../launchers/sln-connector/build.gradle.kts  |  4 +
 .../launchers/sln-connector/nodes.properties  |  2 +
 .../fc/FederatedCatalogServicesExtension.java | 38 ++++++++
 .../fc/directory/InMemoryNodeDirectory.java   | 72 +++++++++++++++
 ...rg.eclipse.edc.spi.system.ServiceExtension |  1 +
 transfer_requests.postman_collection          | 92 +++++++++++++++++++
 9 files changed, 235 insertions(+), 5 deletions(-)
 create mode 100644 sln-connector/launchers/sln-connector/nodes.properties
 create mode 100644 sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java
 create mode 100644 sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java

diff --git a/Readme.md b/Readme.md
index 1c6a366..c1bf743 100644
--- a/Readme.md
+++ b/Readme.md
@@ -4,7 +4,7 @@ This repository contains an [EDC Connector](https://github.com/eclipse-edc/Conne
 designed to connect dataspaces within the SmartLivingNext ecosystem.
 The corresponding dataspace blueprint (helm chart) can be found [here](https://gitlab.int.smartlivingnext.de/data-space/components/helm/dataspace).
 
-The connector's EDC version is currently 0.8.1 and includes extensions for control- and data-plane persistence, a customized policy extension, and authentication via OAuth2.
+The connector's EDC version is currently 0.8.1 and includes extensions for control and data plane persistence, a custom policy extension, a federated catalog extension, and OAuth2-based authentication.
 
 ## Get the Docker Image
 
@@ -102,7 +102,7 @@ Accordingly, you can create policies that allow the use of assets only in a cert
 In the local setup the country is set to `DE` for both connectors. If you want to change
 the `country` claim, navigate to [`http://localhost:8080/admin/master/console/#/dataspace`](http://localhost:8080/admin/master/console/#/dataspace) -> Groups -> EDC-Company2 -> Attributes -> country = BE
 
-You can also create policies that allow (or disallow) the use of assets for a group of countries or participants.
+You can also create policies that allow or disallow (using `"@id": "odrl:isNoneOf"`) the use of assets for a group of countries or participants.
 
 ```json
 {
@@ -121,10 +121,9 @@ You can also create policies that allow (or disallow) the use of assets for a gr
                     "@type": "AtomicConstraint",
                     "leftOperand": "participantId",
                     "operator": {
-                        "@id": "odrl:isPartOf"                      
-                        // "@id": "odrl:isNoneOf"
+                        "@id": "odrl:isPartOf"
                     },
-                    "rightOperand": "company1, company2"  // should be formated as text like this
+                    "rightOperand": "company1, company2"
                 }
             }
         ],
@@ -173,3 +172,7 @@ When defining a contract definition, you must set the `accessPolicyId` and `cont
 ## Postgres Configuration
 
 The persistence modules store the control- and dataplane state in a postgres database. Make sure to apply the corresponding database objects to your postgres database. In the local setup this is already configured for both connectors: [consumer.sql](sln-connector/launchers/sln-connector/postgres/consumer.sql) and [provider.sql](sln-connector/launchers/sln-connector/postgres/provider.sql)
+
+## Federated Catalog Extension
+
+The Federated Catalog, found [here](https://github.com/eclipse-edc/FederatedCatalog), is configured as an extension for this EDC. Essentially, it functions as a crawler, periodically scanning a list of EDC addresses provided in the [nodes.properties](sln-connector/launchers/sln-connector/nodes.properties) file. The Federated Catalog is then exposed through this API: http://edc-provider:19199/catalog/v1alpha/catalog/query or http://edc-consumer:19199/catalog/v1alpha/catalog/query, as configured in the local setup's [docker-compose.yaml](docker-compose.yaml) (see also examples in the [postman collection](transfer_requests.postman_collection)).
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 98b21b9..7a4b82b 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -11,8 +11,10 @@ services:
       - 19193:19193 # management endpoint
       # - 19194:19194 # ds protocol (normaly forwarded, but edcs are in same network)
       - 19291:19291 # public endpoint
+      - 19199:19199 # federated catalog endpoint
     volumes:
       - ./sln-connector/launchers/sln-connector/vault.properties:/resources/vault.properties
+      - ./sln-connector/launchers/sln-connector/nodes.properties:/resources/nodes.properties
     environment:
       # contains public & private key for oauth2
       EDC_VAULT_FILE_PATH: /resources/vault.properties
@@ -24,6 +26,8 @@ services:
       WEB_HTTP_MANAGEMENT_PATH: /management
       WEB_HTTP_PROTOCOL_PORT: 19194
       WEB_HTTP_PROTOCOL_PATH: /protocol
+      WEB_HTTP_CATALOG_PORT: 19199
+      WEB_HTTP_CATALOG_PATH: /catalog
       EDC_PUBLIC_KEY_ALIAS: public-key
       EDC_TRANSFER_PROXY_TOKEN_SIGNER_PRIVATEKEY_ALIAS: private-key
       EDC_TRANSFER_PROXY_TOKEN_VERIFIER_PUBLICKEY_ALIAS: public-key
@@ -48,6 +52,8 @@ services:
       # name of the claim key
       EDC_AGENT_IDENTITY_KEY: participant_id
       EDC_PARTICIPANT_ID: company1
+      #read edc adresses from file
+      EDC_NODES_FILE_PATH: /resources/nodes.properties
   edc-consumer:
     build:
       context: .
@@ -59,8 +65,10 @@ services:
       - 29193:19193 # management endpoint
       # - 29194:19194 # ds protocol (normaly forwarded, but edcs are in same network)
       - 29291:19291 # public endpoint
+      - 29199:19199 # federated catalog endpoint
     volumes:
       - ./sln-connector/launchers/sln-connector/vault.properties:/resources/vault.properties
+      - ./sln-connector/launchers/sln-connector/nodes.properties:/resources/nodes.properties
     environment:
       # contains public & private key for oauth2
       EDC_VAULT_FILE_PATH: /resources/vault.properties
@@ -72,6 +80,8 @@ services:
       WEB_HTTP_MANAGEMENT_PATH: /management
       WEB_HTTP_PROTOCOL_PORT: 19194
       WEB_HTTP_PROTOCOL_PATH: /protocol
+      WEB_HTTP_CATALOG_PORT: 19199
+      WEB_HTTP_CATALOG_PATH: /catalog
       EDC_PUBLIC_KEY_ALIAS: public-key
       EDC_TRANSFER_PROXY_TOKEN_SIGNER_PRIVATEKEY_ALIAS: private-key
       EDC_TRANSFER_PROXY_TOKEN_VERIFIER_PUBLICKEY_ALIAS: public-key
@@ -97,6 +107,8 @@ services:
       # name of the claim key
       EDC_AGENT_IDENTITY_KEY: participant_id
       EDC_PARTICIPANT_ID: company2
+      #read edc adresses from file
+      EDC_NODES_FILE_PATH: /resources/nodes.properties
   http-request-logger:
     build:
       context: ./http-request-logger
diff --git a/sln-connector/gradle/libs.versions.toml b/sln-connector/gradle/libs.versions.toml
index fbaf273..c30aefe 100644
--- a/sln-connector/gradle/libs.versions.toml
+++ b/sln-connector/gradle/libs.versions.toml
@@ -103,5 +103,11 @@ edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-dat
 opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version = "1.41.0" }
 opentelemetry-javaagent = { module = "io.opentelemetry.javaagent:opentelemetry-javaagent", version = "2.7.0" }
 
+# federated catalog
+edc-fc-spi-crawler = { module = "org.eclipse.edc:crawler-spi", version.ref = "edc" }
+edc-fc-core = { module = "org.eclipse.edc:federated-catalog-core", version.ref = "edc" }
+edc-fc-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref = "edc" }
+edc-fc-cache-sql = { module = "org.eclipse.edc:federated-catalog-cache-sql", version.ref = "edc" }
+
 [plugins]
 shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" }
diff --git a/sln-connector/launchers/sln-connector/build.gradle.kts b/sln-connector/launchers/sln-connector/build.gradle.kts
index 00cc2de..10f8653 100644
--- a/sln-connector/launchers/sln-connector/build.gradle.kts
+++ b/sln-connector/launchers/sln-connector/build.gradle.kts
@@ -63,6 +63,10 @@ dependencies {
     implementation(libs.postgresql)
     implementation(libs.edc.transaction.local)
     implementation(libs.edc.transaction.datasource.spi)
+
+    implementation(libs.edc.fc.spi.crawler)
+    runtimeOnly(libs.edc.fc.core)
+    runtimeOnly(libs.edc.fc.api)
 }
 
 application {
diff --git a/sln-connector/launchers/sln-connector/nodes.properties b/sln-connector/launchers/sln-connector/nodes.properties
new file mode 100644
index 0000000..fd648b4
--- /dev/null
+++ b/sln-connector/launchers/sln-connector/nodes.properties
@@ -0,0 +1,2 @@
+company1=http://edc-provider:19194/protocol
+company2=http://edc-consumer:19194/protocol
\ No newline at end of file
diff --git a/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java
new file mode 100644
index 0000000..f32e574
--- /dev/null
+++ b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java
@@ -0,0 +1,38 @@
+/*
+ *  Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.extension.fc;
+
+import org.eclipse.edc.crawler.spi.TargetNodeDirectory;
+import org.eclipse.edc.extension.fc.directory.InMemoryNodeDirectory;
+import org.eclipse.edc.runtime.metamodel.annotation.Provider;
+import org.eclipse.edc.runtime.metamodel.annotation.Setting;
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+
+
+/**
+ * Provides default service implementations for fallback
+ * Omitted {@link org.eclipse.edc.runtime.metamodel.annotation.Extension since there this module already contains {@link FederatedCatalogCacheExtension} }
+ */
+public class FederatedCatalogServicesExtension implements ServiceExtension {
+    @Setting 
+    private static final String NODES_FILE_PATH = "edc.nodes.file.path";
+    
+    @Provider
+    public TargetNodeDirectory defaultNodeDirectory(ServiceExtensionContext context) {
+        var nodesFilePath = context.getConfig().getString(NODES_FILE_PATH);
+        return new InMemoryNodeDirectory(nodesFilePath);
+    }
+} 
diff --git a/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java
new file mode 100644
index 0000000..8ddb352
--- /dev/null
+++ b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java
@@ -0,0 +1,72 @@
+/*
+ *  Copyright (c) 2021 Microsoft Corporation
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Microsoft Corporation - Initial implementation
+ *
+ */
+
+package org.eclipse.edc.extension.fc.directory;
+
+import org.eclipse.edc.crawler.spi.TargetNode;
+import org.eclipse.edc.crawler.spi.TargetNodeDirectory;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+public class InMemoryNodeDirectory implements TargetNodeDirectory {
+   
+    private final Map<String, TargetNode> cache = new ConcurrentHashMap<>();
+    
+ 
+
+    // Constructor with hard-coded nodes
+    public InMemoryNodeDirectory(String nodesFilePath) {
+    
+
+        Properties properties = new Properties();
+        try {
+            // Load the properties file
+            FileInputStream inputStream = new FileInputStream(nodesFilePath);
+            properties.load(inputStream);
+            inputStream.close();
+
+            // Iterate over all key-value pairs in the properties file
+            for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+                String key = (String) entry.getKey();
+                String value = (String) entry.getValue();
+                String nodeName = key + "-node";
+                cache.put(nodeName, new TargetNode(nodeName, key, value, List.of("dataspace-protocol-http")));
+
+
+            }
+
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        // Add more nodes as needed
+    }
+
+    @Override
+    public List<TargetNode> getAll() {
+        return List.copyOf(cache.values()); //never return the internal copy
+    }
+
+    @Override
+    public void insert(TargetNode node) {
+        cache.put(node.id(), node);
+    }
+}
\ No newline at end of file
diff --git a/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension
index 8796842..76fda76 100644
--- a/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension
+++ b/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension
@@ -1,2 +1,3 @@
 org.eclipse.edc.extension.policy.PolicyFunctionsExtension
 org.eclipse.edc.extension.vault.SeedVaultExtension
+org.eclipse.edc.extension.fc.FederatedCatalogServicesExtension
diff --git a/transfer_requests.postman_collection b/transfer_requests.postman_collection
index fb710d0..a1d4888 100644
--- a/transfer_requests.postman_collection
+++ b/transfer_requests.postman_collection
@@ -721,6 +721,68 @@
 				}
 			},
 			"response": []
+		},
+		{
+			"name": "Get Federated Catalog (Provider's View)",
+			"request": {
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\r\n  \"@context\": {\r\n    \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\r\n  },\r\n  \"@type\": \"QuerySpec\"\r\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:19199/catalog/v1alpha/catalog/query",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "19199",
+					"path": [
+						"catalog",
+						"v1alpha",
+						"catalog",
+						"query"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Federated Catalog (Consumer's View)",
+			"request": {
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\r\n  \"@context\": {\r\n    \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\r\n  },\r\n  \"@type\": \"QuerySpec\"\r\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:29199/catalog/v1alpha/catalog/query",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "29199",
+					"path": [
+						"catalog",
+						"v1alpha",
+						"catalog",
+						"query"
+					]
+				}
+			},
+			"response": []
 		}
 	],
 	"event": [
@@ -744,5 +806,35 @@
 				]
 			}
 		}
+	],
+	"variable": [
+		{
+			"key": "fetched_catalog",
+			"value": ""
+		},
+		{
+			"key": "catalog_id",
+			"value": ""
+		},
+		{
+			"key": "permission",
+			"value": ""
+		},
+		{
+			"key": "contract_negotiation_id",
+			"value": ""
+		},
+		{
+			"key": "transfer_process_id",
+			"value": ""
+		},
+		{
+			"key": "transfer_id",
+			"value": ""
+		},
+		{
+			"key": "authCode",
+			"value": ""
+		}
 	]
 }
\ No newline at end of file
-- 
GitLab