From 007e77703f4a88ca4346ac48d5dd7c1298bfd151 Mon Sep 17 00:00:00 2001 From: PhuongTa <duy.ta@materna.group> Date: Mon, 16 Sep 2024 17:20:01 +0200 Subject: [PATCH] add federated catalog extension --- Readme.md | 13 ++- docker-compose.yaml | 12 +++ sln-connector/gradle/libs.versions.toml | 6 ++ .../launchers/sln-connector/build.gradle.kts | 4 + .../launchers/sln-connector/nodes.properties | 2 + .../fc/FederatedCatalogServicesExtension.java | 38 ++++++++ .../fc/directory/InMemoryNodeDirectory.java | 72 +++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + transfer_requests.postman_collection | 92 +++++++++++++++++++ 9 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 sln-connector/launchers/sln-connector/nodes.properties create mode 100644 sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java create mode 100644 sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java diff --git a/Readme.md b/Readme.md index 1c6a366..c1bf743 100644 --- a/Readme.md +++ b/Readme.md @@ -4,7 +4,7 @@ This repository contains an [EDC Connector](https://github.com/eclipse-edc/Conne designed to connect dataspaces within the SmartLivingNext ecosystem. The corresponding dataspace blueprint (helm chart) can be found [here](https://gitlab.int.smartlivingnext.de/data-space/components/helm/dataspace). -The connector's EDC version is currently 0.8.1 and includes extensions for control- and data-plane persistence, a customized policy extension, and authentication via OAuth2. +The connector's EDC version is currently 0.8.1 and includes extensions for control and data plane persistence, a custom policy extension, a federated catalog extension, and OAuth2-based authentication. ## Get the Docker Image @@ -102,7 +102,7 @@ Accordingly, you can create policies that allow the use of assets only in a cert In the local setup the country is set to `DE` for both connectors. If you want to change the `country` claim, navigate to [`http://localhost:8080/admin/master/console/#/dataspace`](http://localhost:8080/admin/master/console/#/dataspace) -> Groups -> EDC-Company2 -> Attributes -> country = BE -You can also create policies that allow (or disallow) the use of assets for a group of countries or participants. +You can also create policies that allow or disallow (using `"@id": "odrl:isNoneOf"`) the use of assets for a group of countries or participants. ```json { @@ -121,10 +121,9 @@ You can also create policies that allow (or disallow) the use of assets for a gr "@type": "AtomicConstraint", "leftOperand": "participantId", "operator": { - "@id": "odrl:isPartOf" - // "@id": "odrl:isNoneOf" + "@id": "odrl:isPartOf" }, - "rightOperand": "company1, company2" // should be formated as text like this + "rightOperand": "company1, company2" } } ], @@ -173,3 +172,7 @@ When defining a contract definition, you must set the `accessPolicyId` and `cont ## Postgres Configuration The persistence modules store the control- and dataplane state in a postgres database. Make sure to apply the corresponding database objects to your postgres database. In the local setup this is already configured for both connectors: [consumer.sql](sln-connector/launchers/sln-connector/postgres/consumer.sql) and [provider.sql](sln-connector/launchers/sln-connector/postgres/provider.sql) + +## Federated Catalog Extension + +The Federated Catalog, found [here](https://github.com/eclipse-edc/FederatedCatalog), is configured as an extension for this EDC. Essentially, it functions as a crawler, periodically scanning a list of EDC addresses provided in the [nodes.properties](sln-connector/launchers/sln-connector/nodes.properties) file. The Federated Catalog is then exposed through this API: http://edc-provider:19199/catalog/v1alpha/catalog/query or http://edc-consumer:19199/catalog/v1alpha/catalog/query, as configured in the local setup's [docker-compose.yaml](docker-compose.yaml) (see also examples in the [postman collection](transfer_requests.postman_collection)). \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 98b21b9..7a4b82b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,8 +11,10 @@ services: - 19193:19193 # management endpoint # - 19194:19194 # ds protocol (normaly forwarded, but edcs are in same network) - 19291:19291 # public endpoint + - 19199:19199 # federated catalog endpoint volumes: - ./sln-connector/launchers/sln-connector/vault.properties:/resources/vault.properties + - ./sln-connector/launchers/sln-connector/nodes.properties:/resources/nodes.properties environment: # contains public & private key for oauth2 EDC_VAULT_FILE_PATH: /resources/vault.properties @@ -24,6 +26,8 @@ services: WEB_HTTP_MANAGEMENT_PATH: /management WEB_HTTP_PROTOCOL_PORT: 19194 WEB_HTTP_PROTOCOL_PATH: /protocol + WEB_HTTP_CATALOG_PORT: 19199 + WEB_HTTP_CATALOG_PATH: /catalog EDC_PUBLIC_KEY_ALIAS: public-key EDC_TRANSFER_PROXY_TOKEN_SIGNER_PRIVATEKEY_ALIAS: private-key EDC_TRANSFER_PROXY_TOKEN_VERIFIER_PUBLICKEY_ALIAS: public-key @@ -48,6 +52,8 @@ services: # name of the claim key EDC_AGENT_IDENTITY_KEY: participant_id EDC_PARTICIPANT_ID: company1 + #read edc adresses from file + EDC_NODES_FILE_PATH: /resources/nodes.properties edc-consumer: build: context: . @@ -59,8 +65,10 @@ services: - 29193:19193 # management endpoint # - 29194:19194 # ds protocol (normaly forwarded, but edcs are in same network) - 29291:19291 # public endpoint + - 29199:19199 # federated catalog endpoint volumes: - ./sln-connector/launchers/sln-connector/vault.properties:/resources/vault.properties + - ./sln-connector/launchers/sln-connector/nodes.properties:/resources/nodes.properties environment: # contains public & private key for oauth2 EDC_VAULT_FILE_PATH: /resources/vault.properties @@ -72,6 +80,8 @@ services: WEB_HTTP_MANAGEMENT_PATH: /management WEB_HTTP_PROTOCOL_PORT: 19194 WEB_HTTP_PROTOCOL_PATH: /protocol + WEB_HTTP_CATALOG_PORT: 19199 + WEB_HTTP_CATALOG_PATH: /catalog EDC_PUBLIC_KEY_ALIAS: public-key EDC_TRANSFER_PROXY_TOKEN_SIGNER_PRIVATEKEY_ALIAS: private-key EDC_TRANSFER_PROXY_TOKEN_VERIFIER_PUBLICKEY_ALIAS: public-key @@ -97,6 +107,8 @@ services: # name of the claim key EDC_AGENT_IDENTITY_KEY: participant_id EDC_PARTICIPANT_ID: company2 + #read edc adresses from file + EDC_NODES_FILE_PATH: /resources/nodes.properties http-request-logger: build: context: ./http-request-logger diff --git a/sln-connector/gradle/libs.versions.toml b/sln-connector/gradle/libs.versions.toml index fbaf273..c30aefe 100644 --- a/sln-connector/gradle/libs.versions.toml +++ b/sln-connector/gradle/libs.versions.toml @@ -103,5 +103,11 @@ edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-dat opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version = "1.41.0" } opentelemetry-javaagent = { module = "io.opentelemetry.javaagent:opentelemetry-javaagent", version = "2.7.0" } +# federated catalog +edc-fc-spi-crawler = { module = "org.eclipse.edc:crawler-spi", version.ref = "edc" } +edc-fc-core = { module = "org.eclipse.edc:federated-catalog-core", version.ref = "edc" } +edc-fc-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref = "edc" } +edc-fc-cache-sql = { module = "org.eclipse.edc:federated-catalog-cache-sql", version.ref = "edc" } + [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/sln-connector/launchers/sln-connector/build.gradle.kts b/sln-connector/launchers/sln-connector/build.gradle.kts index 00cc2de..10f8653 100644 --- a/sln-connector/launchers/sln-connector/build.gradle.kts +++ b/sln-connector/launchers/sln-connector/build.gradle.kts @@ -63,6 +63,10 @@ dependencies { implementation(libs.postgresql) implementation(libs.edc.transaction.local) implementation(libs.edc.transaction.datasource.spi) + + implementation(libs.edc.fc.spi.crawler) + runtimeOnly(libs.edc.fc.core) + runtimeOnly(libs.edc.fc.api) } application { diff --git a/sln-connector/launchers/sln-connector/nodes.properties b/sln-connector/launchers/sln-connector/nodes.properties new file mode 100644 index 0000000..fd648b4 --- /dev/null +++ b/sln-connector/launchers/sln-connector/nodes.properties @@ -0,0 +1,2 @@ +company1=http://edc-provider:19194/protocol +company2=http://edc-consumer:19194/protocol \ No newline at end of file diff --git a/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java new file mode 100644 index 0000000..f32e574 --- /dev/null +++ b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/FederatedCatalogServicesExtension.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.extension.fc; + +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; +import org.eclipse.edc.extension.fc.directory.InMemoryNodeDirectory; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + + +/** + * Provides default service implementations for fallback + * Omitted {@link org.eclipse.edc.runtime.metamodel.annotation.Extension since there this module already contains {@link FederatedCatalogCacheExtension} } + */ +public class FederatedCatalogServicesExtension implements ServiceExtension { + @Setting + private static final String NODES_FILE_PATH = "edc.nodes.file.path"; + + @Provider + public TargetNodeDirectory defaultNodeDirectory(ServiceExtensionContext context) { + var nodesFilePath = context.getConfig().getString(NODES_FILE_PATH); + return new InMemoryNodeDirectory(nodesFilePath); + } +} diff --git a/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java new file mode 100644 index 0000000..8ddb352 --- /dev/null +++ b/sln-connector/launchers/sln-connector/src/main/java/org/eclipse/edc/extension/fc/directory/InMemoryNodeDirectory.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - Initial implementation + * + */ + +package org.eclipse.edc.extension.fc.directory; + +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + + +public class InMemoryNodeDirectory implements TargetNodeDirectory { + + private final Map<String, TargetNode> cache = new ConcurrentHashMap<>(); + + + + // Constructor with hard-coded nodes + public InMemoryNodeDirectory(String nodesFilePath) { + + + Properties properties = new Properties(); + try { + // Load the properties file + FileInputStream inputStream = new FileInputStream(nodesFilePath); + properties.load(inputStream); + inputStream.close(); + + // Iterate over all key-value pairs in the properties file + for (Map.Entry<Object, Object> entry : properties.entrySet()) { + String key = (String) entry.getKey(); + String value = (String) entry.getValue(); + String nodeName = key + "-node"; + cache.put(nodeName, new TargetNode(nodeName, key, value, List.of("dataspace-protocol-http"))); + + + } + + + } catch (IOException e) { + throw new RuntimeException(e); + } + + // Add more nodes as needed + } + + @Override + public List<TargetNode> getAll() { + return List.copyOf(cache.values()); //never return the internal copy + } + + @Override + public void insert(TargetNode node) { + cache.put(node.id(), node); + } +} \ No newline at end of file diff --git a/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension index 8796842..76fda76 100644 --- a/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ b/sln-connector/launchers/sln-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -1,2 +1,3 @@ org.eclipse.edc.extension.policy.PolicyFunctionsExtension org.eclipse.edc.extension.vault.SeedVaultExtension +org.eclipse.edc.extension.fc.FederatedCatalogServicesExtension diff --git a/transfer_requests.postman_collection b/transfer_requests.postman_collection index fb710d0..a1d4888 100644 --- a/transfer_requests.postman_collection +++ b/transfer_requests.postman_collection @@ -721,6 +721,68 @@ } }, "response": [] + }, + { + "name": "Get Federated Catalog (Provider's View)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"@context\": {\r\n \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\r\n },\r\n \"@type\": \"QuerySpec\"\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:19199/catalog/v1alpha/catalog/query", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "19199", + "path": [ + "catalog", + "v1alpha", + "catalog", + "query" + ] + } + }, + "response": [] + }, + { + "name": "Get Federated Catalog (Consumer's View)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"@context\": {\r\n \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\r\n },\r\n \"@type\": \"QuerySpec\"\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:29199/catalog/v1alpha/catalog/query", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "29199", + "path": [ + "catalog", + "v1alpha", + "catalog", + "query" + ] + } + }, + "response": [] } ], "event": [ @@ -744,5 +806,35 @@ ] } } + ], + "variable": [ + { + "key": "fetched_catalog", + "value": "" + }, + { + "key": "catalog_id", + "value": "" + }, + { + "key": "permission", + "value": "" + }, + { + "key": "contract_negotiation_id", + "value": "" + }, + { + "key": "transfer_process_id", + "value": "" + }, + { + "key": "transfer_id", + "value": "" + }, + { + "key": "authCode", + "value": "" + } ] } \ No newline at end of file -- GitLab