From 5f21469a73d849de02824104e2cede751e86bcaa Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 6 Apr 2026 16:42:22 -0500 Subject: [PATCH 1/8] Bump Python version to 3.11 Add Python Requests Library to Dockerfile requirements.txt Update shebang headers to explicitly use python3.11, since it's not the default version for some places these scripts will need to be run. --- Dockerfile | 2 +- comanage_utils.py | 2 +- create_project.py | 3 +-- group_fixup.py | 2 +- osg-comanage-project-usermap.py | 2 +- project_group_setup.py | 2 +- requirements.txt | 4 +++- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 161a98a..51e6ce9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM library/python:3.7-alpine +FROM library/python:3.11-alpine LABEL maintainer OSG Software diff --git a/comanage_utils.py b/comanage_utils.py index 7f5b5b9..374e2ec 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 import os import re diff --git a/create_project.py b/create_project.py index cc447a1..deee7b4 100755 --- a/create_project.py +++ b/create_project.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 import os import re @@ -152,4 +152,3 @@ def main(args): except (RuntimeError, urllib.error.HTTPError) as e: print(e, file=sys.stderr) sys.exit(1) - diff --git a/group_fixup.py b/group_fixup.py index 34d3839..7db24df 100755 --- a/group_fixup.py +++ b/group_fixup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 import os import re diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index bccd3ce..2287d27 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 import os import re diff --git a/project_group_setup.py b/project_group_setup.py index a263e04..f6b9418 100755 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 import os import sys diff --git a/requirements.txt b/requirements.txt index aa2a128..7eb0806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -ldap3~=2.9.1 \ No newline at end of file +ldap3~=2.9.1 +requests>=v2.32.4 + From 8b04c6a3d7898aaef55da305f7ce966c6c8201e5 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 6 Apr 2026 16:52:51 -0500 Subject: [PATCH 2/8] Add Config file for LDAP Server Connections --- comanage_utils.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/comanage_utils.py b/comanage_utils.py index 374e2ec..3823be2 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -4,9 +4,13 @@ import re import json import time +import configparser import urllib.error import urllib.request +from enum import StrEnum +from pathlib import Path from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls +from ldap3.core.exceptions import LDAPException from dataclasses import dataclass #PRODUCTION VALUES @@ -32,6 +36,33 @@ TIMEOUT_BASE = 5 MAX_ATTEMPTS = 5 +# LDAP Search Bases + +# LDAP Server Connection and Search Config, required keys +class LDAP_CONFIG_KEYS(StrEnum): + LDAP_Server_URL = "LDAPServerURl" + LDAP_Search_Base = "SearchBase" + LDAP_User = "User" + LDAP_AuthTok_File = "AuthTokenFile" + +LDAP_CONFIG_USAGE_MESSAGE = f""" +LDAP CONNECTION CONFIG: +File at LDAP_CONFIG_PATH should be in ini format, servers will be attempted in descending order. +An example section of this config file follows: + +--- + +[human_server_name] # arbitrary human label for this server's config +{LDAP_CONFIG_KEYS.LDAP_Server_URL} = ldaps://ldap-replica-1.osg.chtc.io # URL to reach this LDAP server from +{LDAP_CONFIG_KEYS.LDAP_Search_Base} = dc=osg-htc,dc=org # LDAP Search Base +{LDAP_CONFIG_KEYS.LDAP_User} = cn=readonly,ou=system,dc=osg-htc,dc=org # full LDAP user DN +{LDAP_CONFIG_KEYS.LDAP_AuthTok_File} = /etc/ldap-secrets/osg-ldap/authtoken # file containing authtoken for access + +--- + +Config file should contain one such section per LDAP server to communicate with. +""" + GET = "GET" PUT = "PUT" @@ -48,6 +79,13 @@ class URLRequestError(Error): """Class for exceptions due to not being able to fulfill a URLRequest""" pass +class EmptyConfiguration(Error): + """Class for exceptions due to loading an empty Config file, or one where every section lacked the required keys""" + pass + +class NoLDAPResponse(Error): + """Class for exceptions due to being unable to get any request from any configured LDAP servers.""" + pass def getpw(user, passfd, passfile): if ":" in user: @@ -78,6 +116,51 @@ def get_ldap_authtok(ldap_authfile): raise PermissionError return ldap_authtok +def read_ldap_conffile(ldap_conffile_path): + config = configparser.ConfigParser(allow_no_value=True) + + print(f"Attempting to read config from {ldap_conffile_path}") + config.read(ldap_conffile_path) + misconfigured_sections = list() + for section in config.sections(): + for key in LDAP_CONFIG_KEYS: + # All servers must have all required keys for operation + if not config.has_option(section, key) or config.get(section, key) == "": + print(f"Section \"{section}\": required key \"{key}\" missing, ignoring section.") + misconfigured_sections.append(section) + break + # For-Else to only check key values if we know the required ones exist (i.e. we didn't break) + else: + # All server AuthTok files must exist, be files, and not be empty + token_path = Path(config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File)) + try: + if not token_path.exists(): + print(f"Section \"{section}\": AuthToken File missing or non-file, ignoring section.") + misconfigured_sections.append(section) + continue + if token_path.stat().st_size == 0 : + print(f"Section \"{section}\": AuthToken File is empty, ignoring section.") + misconfigured_sections.append(section) + continue + except OSError as e: + print(f"Section \"{section}\": Exception raised while checking AuthTok File: {e}, skipping section.") + misconfigured_sections.append(section) + continue + + for section in misconfigured_sections: + print(f"Dropping section {section}") + config.remove_section(section) + + # if their are no servers in the file that have all required keys + if len(config.sections()) == 0: + #when script needs to say LDAP Config required, raise a EmptyConfiguration error + #usage("LDAP Config File Required") + raise EmptyConfiguration( + f"Config file at {ldap_conffile_path} was empty or all sections lacked required keys." + ) + print(f"Finished reading config from {ldap_conffile_path}") + return config + def mkrequest(method, target, data, endpoint, authstr, **kw): url = os.path.join(endpoint, target) From a4b41c025c506de6df903410d2a6a8ef715ef6c4 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 6 Apr 2026 16:54:03 -0500 Subject: [PATCH 3/8] Add method for Paged, Fallback search for LDAP servers --- comanage_utils.py | 60 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index 3823be2..680ed37 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -258,7 +258,7 @@ def get_datalist(data, listname): return data[listname] if data else [] -class LDAPSearch: +class LDAP_Server: """ Wrapper class for LDAP searches. """ server: Server = None connection: Connection = None @@ -267,11 +267,65 @@ def __init__(self, ldap_server, ldap_user, ldap_authtok): self.server = Server(ldap_server, get_info=ALL) self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - def search(self, ou, filter_str, attrs): - _, _, response, _ = self.connection.search(f"ou={ou},{LDAP_BASE_DN}", filter_str, attributes=attrs) + def search(self, ou, search_base, filter_str, attrs): + # simple paged search + # https://github.com/cannatag/ldap3/blob/7991e67d0a2fb2c1f9cbf832d110ad29fc378f9b/docs/manual/source/standard.rst#L4 + # https://ldap3.readthedocs.io/en/latest/tutorial_searches.html#simple-paged-search + response = self.connection.extend.standard.paged_search( + f"ou={ou},{search_base}", + filter_str, + attributes=attrs, + paged_size=500, + generator=True + ) + return response def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): +# TODO: +# do_ldap_fallback_search, get_ldap_groups, and get_ldap_active_users_and_groups should be a method of the LDAPSearch class +# script calling this lib should init LDAPSearch, then call the method that asks for the info it wants. +# Be able to feed in either one server's config to the LDAPSearch, or a conffile to parse with a list of >=1 LDAP servers to do fallback searches with. + +def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: configparser.ConfigParser): + response = None + + if ldap_config == None: + raise EmptyConfiguration( + "Search Attempted with \"None\" config object." + ) + + for section in ldap_config.sections(): + print(f"Attempting search with server {section}") + try: + server_url = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Server_URL) + search_base = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Search_Base) + search_user = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_User) + authtok_file = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File) + authtok = get_ldap_authtok(authtok_file) + + searcher = LDAP_Server(ldap_server=server_url, ldap_user=search_user, ldap_authtok=authtok) + response = searcher.search(search_ou, search_base, search_filter, attrs) + + #If we get a response from one of the servers, we don't need to check the rest + if not response is None: + print(f"Response found for server {section}.") + break + # Perm issue reading token file + except PermissionError as permError: + print(f"Permission Error when attempting search for {section}: {permError}.") + # Problem getting LDAP Response + except LDAPException as ldapError: + print(f"Exception occurred when attempting search for {section}: {ldapError}.") + continue + + if response is None: + raise NoLDAPResponse( + f"No response found via LDAP servers: {[section for section in ldap_config]}." + ) + + return response + ldap_group_osggids = set() searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) response = searcher.search("groups", "(cn=*)", ["gidNumber"]) From cf178ec275ee2e521db3d6204193d438d1d27978 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 6 Apr 2026 17:04:36 -0500 Subject: [PATCH 4/8] Have LDAP searches use the new setup instead Rearrange flags and usage for project_group_setup.py osg-comanage-project-usermap.py to reflect the changes. Note for reviewer: the definition for the comanage_utils/py get_ldap_groups methods got misplaced during commit cleanup for the PR. --- comanage_utils.py | 22 ++++++++++++++++------ osg-comanage-project-usermap.py | 27 ++++++++++++++------------- project_group_setup.py | 32 ++++++++++++++++---------------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index 680ed37..80eec0e 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -281,7 +281,6 @@ def search(self, ou, search_base, filter_str, attrs): return response -def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): # TODO: # do_ldap_fallback_search, get_ldap_groups, and get_ldap_active_users_and_groups should be a method of the LDAPSearch class # script calling this lib should init LDAPSearch, then call the method that asks for the info it wants. @@ -326,22 +325,33 @@ def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: config return response +def get_ldap_groups(config=None): ldap_group_osggids = set() - searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) - response = searcher.search("groups", "(cn=*)", ["gidNumber"]) + + response = do_ldap_fallback_search( + search_ou="groups", + search_filter="(cn=*)", + attrs=["gidNumber"], + ldap_config=config + ) + for group in response: ldap_group_osggids.add(group["attributes"]["gidNumber"]) return ldap_group_osggids -def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): +def get_ldap_active_users_and_groups(filter_group_name=None, config=None): """ Retrieve a dictionary of active users from LDAP, with their group memberships. """ ldap_active_users = dict() filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") - searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) - response = searcher.search("people", filter_str, ["employeeNumber", "isMemberOf"]) + response = do_ldap_fallback_search( + search_ou="people", + search_filter=filter_str, + attrs=["employeeNumber", "isMemberOf"], + ldap_config=config + ) for person in response: ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", []) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 2287d27..e783b3e 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -24,9 +24,7 @@ OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -s LDAP_SERVER specify LDAP server to read data from - -l LDAP_USER specify LDAP user for reading data from LDAP server - -a ldap_authfile specify path to file to open and read LDAP authtok + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -42,6 +40,9 @@ 2. -d passfd (read from fd) 3. -f passfile (read from file) 4. read from $PASS env var + +{utils.LDAP_CONFIG_USAGE_MESSAGE} + """ def usage(msg=None): @@ -58,10 +59,8 @@ class Options: osg_co_id = OSG_CO_ID outfile = None authstr = None - ldap_server = LDAP_SERVER - ldap_user = LDAP_USER - ldap_authtok = None filtergrp = None + ldap_config = None min_users = 100 # Bail out before updating the file if we have fewer than this many users localmaps = [] @@ -80,7 +79,7 @@ def get_osg_co_groups__map(): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h:n:m:') + ops, args = getopt.getopt(args, 'u:c:l:d:f:g:e:o:h:n:m:') except getopt.GetoptError: usage() @@ -89,15 +88,13 @@ def parse_options(args): passfd = None passfile = None - ldap_authfile = None + ldap_auth_path = None for op, arg in ops: if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) - if op == '-s': options.ldap_server= arg - if op == '-l': options.ldap_user = arg - if op == '-a': ldap_authfile = arg + if op == '-l': ldap_config_path = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg @@ -109,9 +106,13 @@ def parse_options(args): try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) - options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile) except PermissionError: usage("PASS required") + + try: + options.ldap_config = utils.read_ldap_conffile(ldap_config_path) + except utils.EmptyConfiguration: + usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") def _deduplicate_list(items): """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. @@ -120,7 +121,7 @@ def _deduplicate_list(items): return list(dict.fromkeys(items)) def get_osguser_groups(filter_group_name=None): - ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) + ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=options.ldap_config) topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() project_names = topology_projects.keys() diff --git a/project_group_setup.py b/project_group_setup.py index f6b9418..3aaede2 100755 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -24,10 +24,8 @@ -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) - -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) - -s LDAP_SERVER specify LDAP server - -y LDAP_USER specify LDAP server user - -p LDAP authtok specify LDAP server authtok + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -t LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -40,6 +38,9 @@ 2. -d passfd (read from fd) 3. -f passfile (read from file) 4. read from $PASS env var + +{utils.LDAP_CONFIG_USAGE_MESSAGE} + """ @@ -57,12 +58,10 @@ class Options: osg_co_id = OSG_CO_ID ucid = UNIX_CLUSTER_ID provision_target = LDAP_TARGET_ID - ldap_user = LDAP_USER - ldap_server = LDAP_SERVER outfile = None authstr = None - ldap_authtok = None project_gid_startval = PROJECT_GIDS_START + ldap_config = None options = Options() @@ -70,7 +69,7 @@ class Options: def parse_options(args): try: - ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:s:y:h") + ops, args = getopt.getopt(args, "u:c:g:l:t:a:d:f:e:o:h") except getopt.GetoptError: usage() @@ -89,10 +88,10 @@ def parse_options(args): options.osg_co_id = int(arg) if op == "-g": options.ucid = int(arg) - if op == "-l": + if op == "-t": options.provision_target = int(arg) - if op == "-p": - options.ldap_authtok = arg + if op == '-l': + ldap_config_path = arg if op == "-d": passfd = int(arg) if op == "-f": @@ -101,10 +100,6 @@ def parse_options(args): options.endpoint = arg if op == "-o": options.outfile = arg - if op == "-s": - options.ldap_server = arg - if op == "-y": - options.ldap_user = arg try: user, passwd = utils.getpw(options.user, passfd, passfile) @@ -112,6 +107,11 @@ def parse_options(args): except PermissionError: usage("PASS required") + try: + options.ldap_config = utils.read_ldap_conffile(ldap_config_path) + except utils.EmptyConfiguration: + usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") + def append_if_project(project_groups, group): """If this group has a ospoolproject id, and it starts with "Yes-", it's a project""" @@ -182,7 +182,7 @@ def get_projects_needing_cluster_groups(project_groups): def get_projects_needing_provisioning(project_groups): # project groups provisioned in LDAP - ldap_group_osggids = utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok) + ldap_group_osggids = utils.get_ldap_groups(options.ldap_config) try: # All project osggids project_osggids = set( From 6b6e1e9f64d7f40b5c81201f01b3c9f412a66a2f Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 6 Apr 2026 17:08:36 -0500 Subject: [PATCH 5/8] Cleanup + WS Remove / comment old default values for LDAP stuff Whitespace formatting No functional changes --- comanage_utils.py | 28 ++++++++++++++-------------- osg-comanage-project-usermap.py | 2 -- project_group_setup.py | 2 -- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index 80eec0e..3b9971e 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -15,22 +15,18 @@ #PRODUCTION VALUES -PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/" -PRODUCTION_LDAP_SERVER = "ldaps://ldap.cilogon.org" -PRODUCTION_LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" -PRODUCTION_OSG_CO_ID = 7 -PRODUCTION_UNIX_CLUSTER_ID = 1 -PRODUCTION_LDAP_TARGET_ID = 6 -LDAP_BASE_DN = "o=OSG,o=CO,dc=cilogon,dc=org" +# PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/" +# PRODUCTION_OSG_CO_ID = 7 +# PRODUCTION_UNIX_CLUSTER_ID = 1 +# PRODUCTION_LDAP_TARGET_ID = 6 #TEST VALUES -TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/" -TEST_LDAP_SERVER = "ldaps://ldap-test.cilogon.org" -TEST_LDAP_USER ="uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" -TEST_OSG_CO_ID = 8 -TEST_UNIX_CLUSTER_ID = 10 -TEST_LDAP_TARGET_ID = 9 +# TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/" +# TEST_LDAP_SERVER_LIST = ["ldaps://ldap-test.cilogon.org"] +# TEST_OSG_CO_ID = 8 +# TEST_UNIX_CLUSTER_ID = 10 +# TEST_LDAP_TARGET_ID = 9 # Value for the base of the exponential backoff TIMEOUT_BASE = 5 @@ -74,7 +70,6 @@ class Error(Exception): """Base exception class for all exceptions defined""" pass - class URLRequestError(Error): """Class for exceptions due to not being able to fulfill a URLRequest""" pass @@ -87,6 +82,7 @@ class NoLDAPResponse(Error): """Class for exceptions due to being unable to get any request from any configured LDAP servers.""" pass + def getpw(user, passfd, passfile): if ":" in user: user, pw = user.split(":", 1) @@ -116,6 +112,7 @@ def get_ldap_authtok(ldap_authfile): raise PermissionError return ldap_authtok + def read_ldap_conffile(ldap_conffile_path): config = configparser.ConfigParser(allow_no_value=True) @@ -281,6 +278,7 @@ def search(self, ou, search_base, filter_str, attrs): return response + # TODO: # do_ldap_fallback_search, get_ldap_groups, and get_ldap_active_users_and_groups should be a method of the LDAPSearch class # script calling this lib should init LDAPSearch, then call the method that asks for the info it wants. @@ -325,6 +323,7 @@ def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: config return response + def get_ldap_groups(config=None): ldap_group_osggids = set() @@ -425,6 +424,7 @@ def provision_group(gid, provision_target, endpoint, authstr): } return call_api3(POST, path, data, endpoint, authstr) + def provision_group_members(gid, prov_id, endpoint, authstr): data = { "RequestType" : "CoPersonProvisioning", diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index e783b3e..a6a7a45 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -11,8 +11,6 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" -LDAP_SERVER = "ldaps://ldap-test.cilogon.org" -LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 CACHE_FILENAME = "COmanage_Projects_cache.txt" CACHE_LIFETIME_HOURS = 0.5 diff --git a/project_group_setup.py b/project_group_setup.py index 3aaede2..9dc95c5 100755 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -7,8 +7,6 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry.cilogon.org/registry/" -LDAP_SERVER = "ldaps://ldap.cilogon.org" -LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 7 UNIX_CLUSTER_ID = 1 LDAP_TARGET_ID = 6 From b0242b549e20bbe55f873fae96ec6420425ba377 Mon Sep 17 00:00:00 2001 From: williamnswanson Date: Tue, 7 Apr 2026 13:56:27 -0500 Subject: [PATCH 6/8] OSG COmanage LDAP Migration: Change scripts and container to use python 3.9 --- Dockerfile | 2 +- comanage_utils.py | 9 ++++++--- create_project.py | 2 +- group_fixup.py | 2 +- osg-comanage-project-usermap.py | 2 +- project_group_setup.py | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51e6ce9..8ad563f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM library/python:3.11-alpine +FROM library/python:3.9-alpine LABEL maintainer OSG Software diff --git a/comanage_utils.py b/comanage_utils.py index 3b9971e..1cba2a5 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.11 +#!/usr/bin/env python3 import os import re @@ -7,7 +7,7 @@ import configparser import urllib.error import urllib.request -from enum import StrEnum +from enum import Enum from pathlib import Path from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls from ldap3.core.exceptions import LDAPException @@ -35,11 +35,14 @@ # LDAP Search Bases # LDAP Server Connection and Search Config, required keys -class LDAP_CONFIG_KEYS(StrEnum): +class LDAP_CONFIG_KEYS(str, Enum): LDAP_Server_URL = "LDAPServerURl" LDAP_Search_Base = "SearchBase" LDAP_User = "User" LDAP_AuthTok_File = "AuthTokenFile" + + def __str__(self): + return f'{self.value}' LDAP_CONFIG_USAGE_MESSAGE = f""" LDAP CONNECTION CONFIG: diff --git a/create_project.py b/create_project.py index deee7b4..74462de 100755 --- a/create_project.py +++ b/create_project.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.11 +#!/usr/bin/env python3 import os import re diff --git a/group_fixup.py b/group_fixup.py index 7db24df..34d3839 100755 --- a/group_fixup.py +++ b/group_fixup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.11 +#!/usr/bin/env python3 import os import re diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index a6a7a45..b69b28b 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.11 +#!/usr/bin/env python3 import os import re diff --git a/project_group_setup.py b/project_group_setup.py index 9dc95c5..207c2be 100755 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.11 +#!/usr/bin/env python3 import os import sys From 10dfaf83dd25aaac19ab882430dcdb50c9c33382 Mon Sep 17 00:00:00 2001 From: williamnswanson Date: Tue, 7 Apr 2026 13:58:06 -0500 Subject: [PATCH 7/8] WS changes, usage text + magic value notes cleanup --- comanage_utils.py | 5 +---- osg-comanage-project-usermap.py | 23 +++++++++++------------ project_group_setup.py | 21 ++++++++++----------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index 1cba2a5..97919bf 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -23,7 +23,6 @@ #TEST VALUES # TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/" -# TEST_LDAP_SERVER_LIST = ["ldaps://ldap-test.cilogon.org"] # TEST_OSG_CO_ID = 8 # TEST_UNIX_CLUSTER_ID = 10 # TEST_LDAP_TARGET_ID = 9 @@ -32,8 +31,6 @@ TIMEOUT_BASE = 5 MAX_ATTEMPTS = 5 -# LDAP Search Bases - # LDAP Server Connection and Search Config, required keys class LDAP_CONFIG_KEYS(str, Enum): LDAP_Server_URL = "LDAPServerURl" @@ -51,7 +48,7 @@ def __str__(self): --- -[human_server_name] # arbitrary human label for this server's config +[human_server_name] # arbitrary human label for this server's config {LDAP_CONFIG_KEYS.LDAP_Server_URL} = ldaps://ldap-replica-1.osg.chtc.io # URL to reach this LDAP server from {LDAP_CONFIG_KEYS.LDAP_Search_Base} = dc=osg-htc,dc=org # LDAP Search Base {LDAP_CONFIG_KEYS.LDAP_User} = cn=readonly,ou=system,dc=osg-htc,dc=org # full LDAP user DN diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index b69b28b..40fe8a4 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -20,18 +20,17 @@ usage: {SCRIPT} [OPTIONS] OPTIONS: - -u USER[:PASS] specify USER and optionally PASS on command line - -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers - -d passfd specify open fd to read PASS - -f passfile specify path to file to open and read PASS - -e ENDPOINT specify REST endpoint - (default = {ENDPOINT}) - -o outfile specify output file (default: write to stdout) - -g filter_group filter users by group name (eg, 'ap1-login') - -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile - -n min_users Specify minimum number of users required to update the output file (default: 100) - -h display this help text + -u USER[:PASS] specify USER and optionally PASS on command line + -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -d passfd specify open fd to read PASS + -f passfile specify path to file to open and read PASS + -e ENDPOINT specify REST endpoint (default = {ENDPOINT}) + -o outfile specify output file (default: write to stdout) + -g filter_group filter users by group name (eg, 'ap1-login') + -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -n min_users Specify minimum number of users required to update the output file (default: 100) + -h display this help text PASS for USER is taken from the first of: 1. -u USER:PASS diff --git a/project_group_setup.py b/project_group_setup.py index 207c2be..dc953d3 100755 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -19,17 +19,16 @@ usage: [PASS=...] {SCRIPT} [OPTIONS] OPTIONS: - -u USER[:PASS] specify USER and optionally PASS on command line - -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) - -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers - -t LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) - -d passfd specify open fd to read PASS - -f passfile specify path to file to open and read PASS - -e ENDPOINT specify REST endpoint - (default = {ENDPOINT}) - -o outfile specify output file (default: write to stdout) - -h display this help text + -u USER[:PASS] specify USER and optionally PASS on command line + -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -t LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) + -d passfd specify open fd to read PASS + -f passfile specify path to file to open and read PASS + -e ENDPOINT specify REST endpoint (default = {ENDPOINT}) + -o outfile specify output file (default: write to stdout) + -h display this help text PASS for USER is taken from the first of: 1. -u USER:PASS From 1867a2194c9e8d57d525761666abc163a483385f Mon Sep 17 00:00:00 2001 From: williamnswanson Date: Tue, 7 Apr 2026 14:11:45 -0500 Subject: [PATCH 8/8] Change LDAP_Server class to LDAPServer to match naming conventions --- comanage_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index 97919bf..a6d28ec 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -255,7 +255,7 @@ def get_datalist(data, listname): return data[listname] if data else [] -class LDAP_Server: +class LDAPServer: """ Wrapper class for LDAP searches. """ server: Server = None connection: Connection = None @@ -301,7 +301,7 @@ def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: config authtok_file = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File) authtok = get_ldap_authtok(authtok_file) - searcher = LDAP_Server(ldap_server=server_url, ldap_user=search_user, ldap_authtok=authtok) + searcher = LDAPServer(ldap_server=server_url, ldap_user=search_user, ldap_authtok=authtok) response = searcher.search(search_ou, search_base, search_filter, attrs) #If we get a response from one of the servers, we don't need to check the rest