Skip to content

Commit 3847688

Browse files
mwienyamaceay
authored andcommitted
fix: evaluations package with auth aligned
1 parent c2e26dc commit 3847688

3 files changed

Lines changed: 50 additions & 275 deletions

File tree

‎packages/core/ai_core_sdk/credentials.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ def init_conf(profile: str = None):
175175
return config
176176

177177

178-
def _extract_credentials(source: Source, credential_values: List[CredentialsValue], exclude: List[str] = None) \
178+
def extract_credentials(source: Source, credential_values: List[CredentialsValue], exclude: List[str] = None) \
179179
-> Dict[str, str]:
180180
"""Extract all credentials from a source."""
181181
exclude = exclude or []
@@ -188,10 +188,10 @@ def _extract_credentials(source: Source, credential_values: List[CredentialsValu
188188
return credentials
189189

190190

191-
def _resolve_credentials(sources: List[Source], credential_values: List[CredentialsValue]) -> Dict[str, str]:
191+
def resolve_credentials(sources: List[Source], credential_values: List[CredentialsValue]) -> Dict[str, str]:
192192
"""Extract credentials from the first source that has any defined."""
193193
for source in sources:
194-
if credentials := _extract_credentials(source, exclude=['resource_group'], credential_values=credential_values):
194+
if credentials := extract_credentials(source, exclude=['resource_group'], credential_values=credential_values):
195195
logger.debug(f"Using credentials from: {source.name}")
196196
return credentials
197197
raise ValueError("No credentials found in any source")
@@ -309,7 +309,7 @@ def fetch_credentials(profile: str = None, credential_values: List[CredentialsVa
309309
lambda cv, vcap_service = _load_vcap_service_key(): _str_or_none(vcap_service.get(cv.vcap_key, None) if vcap_service and cv.vcap_key else None)),
310310
]
311311

312-
credentials = _resolve_credentials(sources, credential_values)
312+
credentials = resolve_credentials(sources, credential_values)
313313

314314
# Use cert_url as auth_url if present (VCAP provides cert_url for certificate auth)
315315
if 'cert_url' in credentials:
Lines changed: 44 additions & 269 deletions
Original file line numberDiff line numberDiff line change
@@ -1,290 +1,65 @@
11
from __future__ import annotations
22

3-
from typing import Any, Dict, Final, List, Optional, Callable, Tuple
4-
import json
5-
import os
6-
import pathlib
7-
8-
from dataclasses import dataclass
9-
10-
from gen_ai_hub.evaluations.constants import (
11-
AI_CORE_PREFIX,
12-
AUTH_ENDPOINT_SUFFIX,
13-
ENV_VAR_AICORE_CONFIG_FILE,
14-
ENV_VAR_AICORE_PROFILE,
15-
VCAP_AICORE_SERVICE_NAME,
16-
ENV_VAR_VCAP_SERVICES,
17-
ENV_VAR_AICORE_HOME_PATH,
18-
DEFAULT_HOME_PATH,
3+
from typing import Dict, Final, List
4+
5+
from ai_core_sdk.credentials import (
6+
CORE_CREDENTIAL_VALUES,
7+
CredentialsValue,
8+
Service,
9+
Source,
10+
VCAPEnvironment,
11+
extract_credentials as _extract_core_credentials,
12+
fetch_credentials as _fetch_core_credentials,
13+
get_nested_value,
14+
init_conf,
15+
resolve_credentials as _resolve_core_credentials,
16+
resolve_resource_group,
17+
validate_credentials,
1918
)
20-
from gen_ai_hub.evaluations.helpers.logging import get_logger
21-
22-
logger = get_logger()
23-
24-
25-
def get_home() -> str:
26-
return os.environ.get(ENV_VAR_AICORE_HOME_PATH, DEFAULT_HOME_PATH)
27-
28-
29-
def get_nested_value(data_dict, keys: List[str]):
30-
"""
31-
Retrieve a nested value from a dictionary using a list of strings.
32-
33-
:param data_dict: The dictionary to search.
34-
:param keys: A list of strings representing nested keys.
35-
:return: The value associated with the nested keys, or None if not found.
36-
"""
37-
current_value = data_dict
38-
for key in keys:
39-
current_value = current_value[key]
40-
return current_value
41-
42-
43-
@dataclass
44-
class VCAPEnvironment:
45-
services: List[Service]
46-
47-
@classmethod
48-
def from_env(cls, env_var: Optional[str] = None):
49-
env_var = env_var or ENV_VAR_VCAP_SERVICES
50-
env = json.loads(os.environ.get(env_var, '{}'))
51-
return cls.from_dict(env)
52-
53-
@classmethod
54-
def from_dict(cls, env: Dict[str, Any]):
55-
services = [Service(service) for services in env.values() for service in services]
56-
return cls(services=services)
57-
58-
def __getitem__(self, name) -> Service:
59-
return self.get_service(name, exactly_one=True)
60-
61-
def get_service(self, label, exactly_one: bool = True) -> Service:
62-
services = [s for s in self.services if s.label == label]
63-
if exactly_one:
64-
if len(services) == 0:
65-
raise KeyError(f"No service found with label '{label}'.")
66-
return services[0]
67-
else:
68-
return services
69-
70-
def get_service_by_name(self, name, exactly_one: bool = True) -> Service:
71-
services = [s for s in self.services if s.name == name]
72-
if exactly_one:
73-
if len(services) == 0:
74-
raise KeyError(f"No service found with name '{name}'.")
75-
return services[0]
76-
else:
77-
return services
78-
79-
80-
NoDefault = object()
81-
82-
83-
class Service:
84-
85-
def __init__(self, env: Dict[str, Any]):
86-
self._env = env
87-
88-
@property
89-
def label(self) -> Optional[str]:
90-
return self._env.get('label')
91-
92-
@property
93-
def name(self) -> Optional[str]:
94-
return self._env.get('name')
95-
96-
def __getitem__(self, key):
97-
return self.get(key)
98-
99-
def get(self, key, default=NoDefault):
100-
if isinstance(key, str):
101-
key_splitted = key.split('.')
102-
else:
103-
key_splitted = key
104-
try:
105-
return get_nested_value(self._env, key_splitted) or default
106-
except KeyError:
107-
if default is NoDefault:
108-
raise KeyError(f"Key '{key}' not found in service '{self.name}'.")
109-
return default
110-
111-
112-
@dataclass
113-
class CredentialsValue:
114-
name: str
115-
vcap_key: Optional[Tuple[str, ...]] = None
116-
transform_fn: Optional[Callable] = None
117-
118-
119-
@dataclass
120-
class Source:
121-
name: str
122-
get: Callable[[CredentialsValue], Optional[str]]
123-
19+
from ai_core_sdk.helpers import get_home
20+
21+
# Re-exported for backward compatibility: these are generic and fully reused from ai_core_sdk.credentials.
22+
__all__ = [
23+
"CredentialsValue",
24+
"Service",
25+
"Source",
26+
"VCAPEnvironment",
27+
"EVAL_CREDENTIAL_VALUES",
28+
"extract_credentials",
29+
"fetch_credentials",
30+
"get_home",
31+
"get_nested_value",
32+
"init_conf",
33+
"resolve_credentials",
34+
"resolve_resource_group",
35+
"validate_credentials",
36+
]
12437

125-
CREDENTIAL_VALUES: Final[List[CredentialsValue]] = [
126-
CredentialsValue(name='client_id', vcap_key=('credentials', 'clientid')),
127-
CredentialsValue(name='client_secret', vcap_key=('credentials', 'clientsecret')),
128-
CredentialsValue(name='auth_url',
129-
vcap_key=('credentials', 'url'),
130-
transform_fn=lambda url: url.rstrip('/') +
131-
('' if url.endswith(AUTH_ENDPOINT_SUFFIX) else AUTH_ENDPOINT_SUFFIX)),
132-
CredentialsValue(name='base_url',
133-
vcap_key=('credentials', 'serviceurls', 'AI_API_URL'),
134-
transform_fn=lambda url: url.rstrip('/') + ('' if url.endswith('/v2') else '/v2')),
135-
CredentialsValue(name='resource_group'),
136-
CredentialsValue(name='cert_url', vcap_key=('credentials', 'certurl'),
137-
transform_fn=lambda url: url.rstrip('/') +
138-
('' if url.endswith(AUTH_ENDPOINT_SUFFIX) else AUTH_ENDPOINT_SUFFIX)),
139-
# Even though the certificate and key in VCAP_SERVICES are not file paths, the names are defined this way in order
140-
# to keep it compatible with the config names. It'll be handled in fetch_credentials function.
141-
CredentialsValue(name='cert_file_path'),
142-
CredentialsValue(name='key_file_path'),
143-
CredentialsValue(name='cert_str', vcap_key=('credentials', 'certificate'),
144-
transform_fn=lambda cert_str: cert_str.replace('\\n', '\n')),
145-
CredentialsValue(name='key_str', vcap_key=('credentials', 'key'),
146-
transform_fn=lambda key_str: key_str.replace('\\n', '\n')),
147-
# Currently supporting only the AWS creds, would need to extend to other hyperscalers in future.
38+
# Extends the core credential values with evaluation-specific ones.
39+
# Currently supporting only the AWS creds, would need to extend to other hyperscalers in future.
40+
EVAL_CREDENTIAL_VALUES: Final[List[CredentialsValue]] = CORE_CREDENTIAL_VALUES + [
14841
CredentialsValue(name='aws_access_key_id'),
14942
CredentialsValue(name='aws_secret_access_key'),
15043
CredentialsValue(name='orchestration_url'),
15144
CredentialsValue(name='input_object_store_secret_name'),
15245
]
15346

15447

155-
def init_conf(profile: str = None):
156-
# Read configuration from ${AICORE_HOME}/config_<profile>.json.
157-
home = pathlib.Path(get_home())
158-
profile = profile or os.environ.get(ENV_VAR_AICORE_PROFILE)
159-
profile_config_file = f'config_{profile}.json'
160-
direct_config_file = pathlib.Path(os.getenv(ENV_VAR_AICORE_CONFIG_FILE)) if os.getenv(ENV_VAR_AICORE_CONFIG_FILE) else None
161-
path_to_config = (direct_config_file or
162-
(home / ('config.json' if profile in ('default', '', None) else profile_config_file)))
163-
config = {}
164-
if path_to_config.exists():
165-
logger.debug('Config file path %s', path_to_config)
166-
try:
167-
with path_to_config.open(encoding='utf-8') as f:
168-
return json.load(f)
169-
except json.decoder.JSONDecodeError:
170-
raise KeyError(f'{path_to_config} is not a valid json file. Please fix or remove it!')
171-
except PermissionError as e:
172-
logger.warning("Permission denied when trying to read config file '%s'. File ignored.", path_to_config)
173-
return config
174-
elif profile:
175-
raise FileNotFoundError(f"Unable to locate profile config file '{profile_config_file}' "
176-
f"in AICORE_HOME '{home}')")
177-
return config
178-
179-
18048
def extract_credentials(source: Source, exclude: List[str] = None) -> Dict[str, str]:
181-
"""Extract all credentials from a source."""
182-
exclude = exclude or []
183-
credentials = {}
184-
for cv in CREDENTIAL_VALUES:
185-
if cv.name in exclude:
186-
continue
187-
if value := source.get(cv):
188-
credentials[cv.name] = cv.transform_fn(value) if cv.transform_fn else value
189-
return credentials
49+
"""Extract all evaluation credentials from a source."""
50+
return _extract_core_credentials(source, credential_values=EVAL_CREDENTIAL_VALUES, exclude=exclude)
19051

19152

19253
def resolve_credentials(sources: List[Source]) -> Dict[str, str]:
193-
"""Extract credentials from the first source that has any defined."""
194-
for source in sources:
195-
if credentials := extract_credentials(source, exclude=['resource_group']):
196-
logger.debug(f"Using credentials from: {source.name}")
197-
return credentials
198-
raise ValueError("No credentials found in any source")
199-
200-
201-
def resolve_resource_group(sources: List[Source]) -> Optional[str]:
202-
"""Find resource_group from the first source that defines it."""
203-
rg_cred = CredentialsValue(name='resource_group')
204-
for source in sources:
205-
if value := source.get(rg_cred):
206-
logger.debug("Using resource_group '%s' from: %s", value, source.name)
207-
return value
208-
logger.debug("No resource_group found in any source")
209-
return None
210-
211-
212-
def validate_credentials(credentials: Dict[str, str]) -> None:
213-
"""Validate that we have a complete authentication method."""
214-
required_base = {'client_id', 'auth_url', 'base_url'}
215-
216-
# Check which auth method we have
217-
has_client_secret = 'client_secret' in credentials
218-
has_cert_files = 'cert_file_path' in credentials and 'key_file_path' in credentials
219-
has_cert_strings = 'cert_str' in credentials and 'key_str' in credentials
220-
221-
# Must have exactly one auth method
222-
auth_methods = sum([has_client_secret, has_cert_files, has_cert_strings])
223-
224-
if auth_methods == 0:
225-
raise ValueError(
226-
"No authentication method found. Must provide one of:\n"
227-
"1. client_secret\n"
228-
"2. cert_file_path AND key_file_path\n"
229-
"3. cert_str AND key_str"
230-
)
231-
232-
if auth_methods > 1:
233-
raise ValueError(
234-
"Multiple authentication methods found. Please provide only one of:\n"
235-
"1. client_secret\n"
236-
"2. cert_file_path AND key_file_path\n"
237-
"3. cert_str AND key_str"
238-
)
239-
240-
# Check required base fields
241-
missing = required_base - set(credentials.keys())
242-
if missing:
243-
raise ValueError(f"Missing required credentials: {missing}")
244-
245-
246-
def _str_or_none(value) -> Optional[str]:
247-
return str(value) if value else None
54+
"""Extract evaluation credentials from the first source that has any defined."""
55+
return _resolve_core_credentials(sources, credential_values=EVAL_CREDENTIAL_VALUES)
24856

24957

25058
def fetch_credentials(profile: str = None, **kwargs) -> Dict[str, str]:
25159
"""
252-
Fetch credentials from a single source based on precedence.
253-
254-
Precedence order: kwargs > environment variables > config file > VCAP service
60+
Fetch evaluation credentials from a single source based on precedence.
25561
256-
Once a source is selected (first one with any credential), all credentials
257-
come from that source only. Resource group is an exception and follows
258-
precedence independently.
62+
Precedence order: kwargs > AICORE_SERVICE_KEY > environment variables > config file > VCAP service
63+
(see ai_core_sdk.credentials.fetch_credentials for the full behavior).
25964
"""
260-
config = init_conf(profile=profile)
261-
262-
try:
263-
vcap_service = VCAPEnvironment.from_env()[VCAP_AICORE_SERVICE_NAME]
264-
except KeyError:
265-
vcap_service = None
266-
267-
sources = [
268-
Source("kwargs",
269-
lambda cv: _str_or_none(kwargs.get(cv.name))),
270-
Source("environment variables",
271-
lambda cv: _str_or_none(os.environ.get(f'{AI_CORE_PREFIX}_{cv.name.upper()}'))),
272-
Source("config file",
273-
lambda cv: _str_or_none(config.get(f'{AI_CORE_PREFIX}_{cv.name.upper()}'))),
274-
Source("VCAP service",
275-
lambda cv: _str_or_none(vcap_service.get(cv.vcap_key, None) if vcap_service and cv.vcap_key else None)),
276-
]
277-
278-
credentials = resolve_credentials(sources)
279-
280-
# Use cert_url as auth_url if present (VCAP provides cert_url for certificate auth)
281-
if 'cert_url' in credentials:
282-
credentials['auth_url'] = credentials.pop('cert_url')
283-
284-
validate_credentials(credentials)
285-
286-
resource_group = resolve_resource_group(sources)
287-
if resource_group:
288-
credentials['resource_group'] = resource_group
289-
290-
return credentials
65+
return _fetch_core_credentials(profile=profile, credential_values=EVAL_CREDENTIAL_VALUES, **kwargs)

‎packages/gen/tests/evaluations/test_credentials.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -320,14 +320,14 @@ def test_fetch_credentials_from_env(self):
320320
"AICORE_AUTH_URL": "https://auth.com/oauth/token",
321321
"AICORE_BASE_URL": "https://api.com/v2",
322322
},
323-
), patch("gen_ai_hub.evaluations.credentials.init_conf", return_value={}):
323+
), patch("ai_core_sdk.credentials.init_conf", return_value={}):
324324
result = fetch_credentials()
325325
self.assertEqual(result["client_id"], "env-client")
326326
self.assertEqual(result["client_secret"], "env-secret")
327327

328328
def test_fetch_credentials_cert_url_becomes_auth_url(self):
329329
with patch(
330-
"gen_ai_hub.evaluations.credentials.init_conf", return_value={}
330+
"ai_core_sdk.credentials.init_conf", return_value={}
331331
), patch.dict(
332332
os.environ,
333333
{

0 commit comments

Comments
 (0)