Move device definitions from Tableland to the R2 template catalog and narrow VIN decodes to one trim

charts/device-definitions-api/templates/secret.yaml

@@ -6,17 +6,14 @@ metadata:
 spec:
   data:
   - remoteRef:
-      key: {{ .Release.Namespace }}/definitions/ethereum/rpc_url
-    secretKey: ETHEREUM_RPC_URL
+      key: {{ .Release.Namespace }}/definitions/worker/token
+    secretKey: DEFINITIONS_WORKER_TOKEN
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/aws/access_key_id
     secretKey: AWS_ACCESS_KEY_ID
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/aws/secret_access_key
     secretKey: AWS_SECRET_ACCESS_KEY
-  - remoteRef:
-      key: {{ .Release.Namespace }}/definitions/kms/key_id
-    secretKey: KMS_KEY_ID
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/db/host
     secretKey: DB_HOST
@@ -77,9 +74,6 @@ spec:
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/typesense/apiurl
     secretKey: SEARCH_SERVICE_API_URL
-  - remoteRef:
-      key: {{ .Release.Namespace }}/definitions/polygonscan/apikey
-    secretKey: POLYGON_SCAN_API_KEY
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/googlesheets/credentials
     secretKey: GOOGLE_SHEETS_CREDENTIALS

charts/device-definitions-api/values-prod.yaml

@@ -20,10 +20,13 @@ env:
   REDIS_CLUSTER_MODE: true
   REDIS_TLS: true
   KAFKA_BROKERS: kafka-prod-dimo-kafka-kafka-brokers:9092
-  ETHEREUM_SEND_TRANSACTION: true
-  TABLELAND_API_GATEWAY: http://tableland-node-prod.prod.svc.cluster.local:8080/
-  ETHEREUM_REGISTRY_ADDRESS: '0xFA8beC73cebB9D88FF88a2f75E7D7312f2Fd39EC'
-  SEARCH_SERVICE_DEVICE_DEFINITION_INDEX: definitions_prod
+  # One hostname on purpose: definitions-worker is what serves
+  # definitions.dimo.org, so there is no separate origin to read through.
+  # A read that must not be served from the edge cache misses it by cache
+  # key instead -- see fetchTemplateDocFresh's cache-busting parameter.
+  DEFINITIONS_CATALOG_URL: https://definitions.dimo.org
+  DEFINITIONS_WORKER_URL: https://definitions.dimo.org
+  SEARCH_SERVICE_DEVICE_DEFINITION_INDEX: definitions_v2_prod
   IDENTITY_API_URL: https://identity-api.dimo.zone/query
 ingress:
   enabled: true
@@ -58,9 +61,4 @@ tolerations: []
 affinity: {}
 podDisruptionBudget:
   minAvailable: 1
-jobs:
-  - name: device-definitions-search-sync
-    schedule: 0 13 * * *
-    args:
-      - '-c'
-      - /device-definitions-api sync-device-definitions-search; CODE=$?; echo "daily device definitions typesense search sync completed"; wget -q --post-data "hello=shutdown" http://localhost:4191/shutdown &> /dev/null; exit $CODE;
+jobs: []

charts/device-definitions-api/values.yaml

@@ -51,10 +51,13 @@ env:
   VINCARIO_API_URL: https://api.vindecoder.eu/3.2
   DAT_GROUP_URL: https://www.datgroup.com/myClaim/soap/v2/VehicleIdentificationService
   PRIVATE_KEY_MODE: false
-  ETHEREUM_SEND_TRANSACTION: true
-  TABLELAND_API_GATEWAY: http://tableland-node-dev.dev.svc.cluster.local:8080/
-  ETHEREUM_REGISTRY_ADDRESS: '0x5eAA326fB2fc97fAcCe6A79A304876daD0F2e96c'
-  SEARCH_SERVICE_DEVICE_DEFINITION_INDEX: definitions_dev
+  # One hostname on purpose: definitions-worker is what serves
+  # definitions.dev.dimo.org, so there is no separate origin to read through.
+  # A read that must not be served from the edge cache misses it by cache
+  # key instead -- see fetchTemplateDocFresh's cache-busting parameter.
+  DEFINITIONS_CATALOG_URL: https://definitions.dev.dimo.org
+  DEFINITIONS_WORKER_URL: https://definitions.dev.dimo.org
+  SEARCH_SERVICE_DEVICE_DEFINITION_INDEX: definitions_v2_dev
   IDENTITY_API_URL: https://identity-api.dev.dimo.zone/query
 service:
   type: ClusterIP
@@ -119,9 +122,4 @@ serviceMonitor:
   path: /metrics
   port: mon-http
   interval: 30s
-jobs:
-  - name: search-index-load
-    schedule: 0 0 * * *
-    args:
-      - '-c'
-      - /device-definitions-api sync-device-definitions-search; CODE=$?; echo "weekly typesense mmy search index sync completed"; wget -q --post-data "hello=shutdown" http://localhost:4191/shutdown &> /dev/null; exit $CODE;
+jobs: []

cmd/device-definitions-api/add_vin.go

@@ -14,8 +14,6 @@ import (
 
 	"github.com/DIMO-Network/device-definitions-api/internal/core/common"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
-	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/sender"
-	"github.com/ethereum/go-ethereum/ethclient"
 
 	"github.com/DIMO-Network/device-definitions-api/internal/config"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/db/models"
@@ -32,8 +30,6 @@ import (
 type addVINCmd struct {
 	logger   zerolog.Logger
 	settings config.Settings
-
-	sender sender.Sender
 }
 
 func (*addVINCmd) Name() string     { return "addvin" }
@@ -61,16 +57,7 @@ func (p *addVINCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interface
 	pdb := db.NewDbConnectionFromSettings(ctx, &p.settings.DB, true)
 	pdb.WaitForDB(p.logger)
 
-	ethClient, err := ethclient.Dial(p.settings.EthereumRPCURL.String())
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
-	}
-
-	chainID, err := ethClient.ChainID(ctx)
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
-	}
-	onChainSvc := gateways.NewDeviceDefinitionOnChainService(&p.settings, &p.logger, ethClient, chainID, p.sender, pdb.DBS)
+	catalogSvc := gateways.NewDeviceDefinitionCatalogService(&p.settings, &p.logger)
 
 	vinDecodeNumber, err := models.FindVinNumber(ctx, pdb.DBS().Reader, vin)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -117,18 +104,30 @@ func (p *addVINCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interface
 		return subcommands.ExitFailure
 	}
 
-	manufacturer, err := onChainSvc.GetManufacturer(stringutils.SlugString(wmi.ManufacturerName))
+	manufacturer, err := catalogSvc.GetManufacturer(stringutils.SlugString(wmi.ManufacturerName))
 	if err != nil {
 		fmt.Println(err.Error())
 		return subcommands.ExitFailure
 	}
-	definitionID := common.DeviceDefinitionSlug(stringutils.SlugString(wmi.ManufacturerName), stringutils.SlugString(model), int16(vinNumber.Year))
-	deviceDefinition, err := onChainSvc.GetDefinitionTableland(ctx, big.NewInt(int64(manufacturer.TokenID)), definitionID)
+	definitionID, err := common.DeviceDefinitionSlug(stringutils.SlugString(wmi.ManufacturerName), stringutils.SlugString(model), int16(vinNumber.Year))
+	if err != nil {
+		// No template can exist at this id, so the lookup below would 404 and
+		// the row would point at a definition that can never be created.
+		fmt.Println(err.Error())
+		return subcommands.ExitFailure
+	}
+	deviceDefinition, err := catalogSvc.GetDefinition(ctx, big.NewInt(int64(manufacturer.TokenID)), definitionID)
 
 	if err != nil {
 		fmt.Println(err.Error() + " " + model + " " + strconv.Itoa(vinNumber.Year))
 		return subcommands.ExitFailure
 	}
+	if deviceDefinition == nil {
+		// (nil, nil) is returned both for a missing definition and for one
+		// owned by a different manufacturer; dereferencing panics.
+		fmt.Println("no device definition found for " + definitionID)
+		return subcommands.ExitFailure
+	}
 	vinNumber.DefinitionID = deviceDefinition.ID
 
 	err = vinNumber.Insert(ctx, pdb.DBS().Writer, boil.Infer())

cmd/device-definitions-api/add_vins_csv.go

@@ -13,7 +13,6 @@ import (
 	"github.com/DIMO-Network/device-definitions-api/internal/config"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/db/models"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
-	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/sender"
 	"github.com/DIMO-Network/shared/pkg/db"
 	"github.com/DIMO-Network/shared/pkg/logfields"
 	vinutils "github.com/DIMO-Network/shared/pkg/vin"
@@ -28,7 +27,6 @@ type addVINsCSVCmd struct {
 	logger   zerolog.Logger
 	settings config.Settings
 
-	sender   sender.Sender
 	identity gateways.IdentityAPI
 }
 

cmd/device-definitions-api/decode_vin.go

@@ -16,7 +16,6 @@ import (
 	vinutil "github.com/DIMO-Network/shared/pkg/vin"
 	"github.com/aarondl/null/v8"
 	"github.com/aarondl/sqlboiler/v4/boil"
-	"github.com/ethereum/go-ethereum/ethclient"
 
 	"github.com/goccy/go-json"
 
@@ -181,7 +180,16 @@ func (p *decodeVINCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interf
 				dbVin.ManufacturerName = vinInfo.Make
 			}
 			dbVin.DatgroupData = null.JSONFrom(vinInfo.Raw)
-			dbVin.DefinitionID = common.DeviceDefinitionSlug(vinInfo.Make, vinInfo.Model, int16(vinInfo.Year))
+			definitionID, errID := common.DeviceDefinitionSlug(vinInfo.Make, vinInfo.Model, int16(vinInfo.Year))
+			if errID != nil {
+				// Inserting the row anyway is what the validator's contract
+				// forbids: every later DecodeVIN of this VIN would read the
+				// cached row, look its definition up in the catalog, 404
+				// forever, and answer with an empty trim and match quality.
+				fmt.Println("skipping, no definition id can be built: " + errID.Error())
+				continue
+			}
+			dbVin.DefinitionID = definitionID
 			dbVin.DecodeProvider = null.StringFrom(string(vinInfo.Source))
 			// todo future change to add field with StyleName
 
@@ -254,7 +262,7 @@ func readVINFile(filename string) ([]string, error) {
 	return values, nil
 }
 
-func instantiateVINDecodingSvc(ctx context.Context, settings *config.Settings, logger *zerolog.Logger, pdb db.Store) services.VINDecodingService {
+func instantiateVINDecodingSvc(_ context.Context, settings *config.Settings, logger *zerolog.Logger, pdb db.Store) services.VINDecodingService {
 	datAPI := gateways.NewDATGroupAPIService(settings, logger)
 	drivlyAPI := gateways.NewDrivlyAPIService(settings)
 	vincarioAPI := gateways.NewVincarioAPIService(settings, logger)
@@ -265,21 +273,7 @@ func instantiateVINDecodingSvc(ctx context.Context, settings *config.Settings, l
 		return services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, nil, datAPI, pdb.DBS, jp17vinAPI, carvxAPI, elevaAPI)
 	}
 
-	send, err := createSender(ctx, settings, logger)
-	if err != nil {
-		logger.Fatal().Err(err).Msg("Failed to create sender.")
-	}
-
-	ethClient, err := ethclient.Dial(settings.EthereumRPCURL.String())
-	if err != nil {
-		logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
-	}
-
-	chainID, err := ethClient.ChainID(ctx)
-	if err != nil {
-		logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
-	}
-	deviceDefinitionOnChainService := gateways.NewDeviceDefinitionOnChainService(settings, logger, ethClient, chainID, send, pdb.DBS)
+	deviceDefinitionCatalogService := gateways.NewDeviceDefinitionCatalogService(settings, logger)
 
-	return services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, deviceDefinitionOnChainService, datAPI, pdb.DBS, jp17vinAPI, carvxAPI, elevaAPI)
+	return services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, deviceDefinitionCatalogService, datAPI, pdb.DBS, jp17vinAPI, carvxAPI, elevaAPI)
 }