diff --git a/Dockerfile b/Dockerfile index 161a98a..8ad563f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM library/python:3.7-alpine +FROM library/python:3.9-alpine LABEL maintainer OSG Software diff --git a/comanage_utils.py b/comanage_utils.py index 7f5b5b9..a6d28ec 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -4,34 +4,61 @@ import re import json import time +import configparser import urllib.error import urllib.request +from enum import Enum +from pathlib import Path from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls +from ldap3.core.exceptions import LDAPException from dataclasses import dataclass #PRODUCTION VALUES -PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/" -PRODUCTION_LDAP_SERVER = "ldaps://ldap.cilogon.org" -PRODUCTION_LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" -PRODUCTION_OSG_CO_ID = 7 -PRODUCTION_UNIX_CLUSTER_ID = 1 -PRODUCTION_LDAP_TARGET_ID = 6 -LDAP_BASE_DN = "o=OSG,o=CO,dc=cilogon,dc=org" +# PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/" +# PRODUCTION_OSG_CO_ID = 7 +# PRODUCTION_UNIX_CLUSTER_ID = 1 +# PRODUCTION_LDAP_TARGET_ID = 6 #TEST VALUES -TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/" -TEST_LDAP_SERVER = "ldaps://ldap-test.cilogon.org" -TEST_LDAP_USER ="uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" -TEST_OSG_CO_ID = 8 -TEST_UNIX_CLUSTER_ID = 10 -TEST_LDAP_TARGET_ID = 9 +# TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/" +# TEST_OSG_CO_ID = 8 +# TEST_UNIX_CLUSTER_ID = 10 +# TEST_LDAP_TARGET_ID = 9 # Value for the base of the exponential backoff TIMEOUT_BASE = 5 MAX_ATTEMPTS = 5 +# LDAP Server Connection and Search Config, required keys +class LDAP_CONFIG_KEYS(str, Enum): + LDAP_Server_URL = "LDAPServerURl" + LDAP_Search_Base = "SearchBase" + LDAP_User = "User" + LDAP_AuthTok_File = "AuthTokenFile" + + def __str__(self): + return f'{self.value}' + +LDAP_CONFIG_USAGE_MESSAGE = f""" +LDAP CONNECTION CONFIG: +File at LDAP_CONFIG_PATH should be in ini format, servers will be attempted in descending order. +An example section of this config file follows: + +--- + +[human_server_name] # arbitrary human label for this server's config +{LDAP_CONFIG_KEYS.LDAP_Server_URL} = ldaps://ldap-replica-1.osg.chtc.io # URL to reach this LDAP server from +{LDAP_CONFIG_KEYS.LDAP_Search_Base} = dc=osg-htc,dc=org # LDAP Search Base +{LDAP_CONFIG_KEYS.LDAP_User} = cn=readonly,ou=system,dc=osg-htc,dc=org # full LDAP user DN +{LDAP_CONFIG_KEYS.LDAP_AuthTok_File} = /etc/ldap-secrets/osg-ldap/authtoken # file containing authtoken for access + +--- + +Config file should contain one such section per LDAP server to communicate with. +""" + GET = "GET" PUT = "PUT" @@ -43,11 +70,18 @@ class Error(Exception): """Base exception class for all exceptions defined""" pass - class URLRequestError(Error): """Class for exceptions due to not being able to fulfill a URLRequest""" pass +class EmptyConfiguration(Error): + """Class for exceptions due to loading an empty Config file, or one where every section lacked the required keys""" + pass + +class NoLDAPResponse(Error): + """Class for exceptions due to being unable to get any request from any configured LDAP servers.""" + pass + def getpw(user, passfd, passfile): if ":" in user: @@ -79,6 +113,52 @@ def get_ldap_authtok(ldap_authfile): return ldap_authtok +def read_ldap_conffile(ldap_conffile_path): + config = configparser.ConfigParser(allow_no_value=True) + + print(f"Attempting to read config from {ldap_conffile_path}") + config.read(ldap_conffile_path) + misconfigured_sections = list() + for section in config.sections(): + for key in LDAP_CONFIG_KEYS: + # All servers must have all required keys for operation + if not config.has_option(section, key) or config.get(section, key) == "": + print(f"Section \"{section}\": required key \"{key}\" missing, ignoring section.") + misconfigured_sections.append(section) + break + # For-Else to only check key values if we know the required ones exist (i.e. we didn't break) + else: + # All server AuthTok files must exist, be files, and not be empty + token_path = Path(config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File)) + try: + if not token_path.exists(): + print(f"Section \"{section}\": AuthToken File missing or non-file, ignoring section.") + misconfigured_sections.append(section) + continue + if token_path.stat().st_size == 0 : + print(f"Section \"{section}\": AuthToken File is empty, ignoring section.") + misconfigured_sections.append(section) + continue + except OSError as e: + print(f"Section \"{section}\": Exception raised while checking AuthTok File: {e}, skipping section.") + misconfigured_sections.append(section) + continue + + for section in misconfigured_sections: + print(f"Dropping section {section}") + config.remove_section(section) + + # if their are no servers in the file that have all required keys + if len(config.sections()) == 0: + #when script needs to say LDAP Config required, raise a EmptyConfiguration error + #usage("LDAP Config File Required") + raise EmptyConfiguration( + f"Config file at {ldap_conffile_path} was empty or all sections lacked required keys." + ) + print(f"Finished reading config from {ldap_conffile_path}") + return config + + def mkrequest(method, target, data, endpoint, authstr, **kw): url = os.path.join(endpoint, target) if kw: @@ -175,7 +255,7 @@ def get_datalist(data, listname): return data[listname] if data else [] -class LDAPSearch: +class LDAPServer: """ Wrapper class for LDAP searches. """ server: Server = None connection: Connection = None @@ -184,27 +264,93 @@ def __init__(self, ldap_server, ldap_user, ldap_authtok): self.server = Server(ldap_server, get_info=ALL) self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - def search(self, ou, filter_str, attrs): - _, _, response, _ = self.connection.search(f"ou={ou},{LDAP_BASE_DN}", filter_str, attributes=attrs) + def search(self, ou, search_base, filter_str, attrs): + # simple paged search + # https://github.com/cannatag/ldap3/blob/7991e67d0a2fb2c1f9cbf832d110ad29fc378f9b/docs/manual/source/standard.rst#L4 + # https://ldap3.readthedocs.io/en/latest/tutorial_searches.html#simple-paged-search + response = self.connection.extend.standard.paged_search( + f"ou={ou},{search_base}", + filter_str, + attributes=attrs, + paged_size=500, + generator=True + ) + return response -def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): + +# TODO: +# do_ldap_fallback_search, get_ldap_groups, and get_ldap_active_users_and_groups should be a method of the LDAPSearch class +# script calling this lib should init LDAPSearch, then call the method that asks for the info it wants. +# Be able to feed in either one server's config to the LDAPSearch, or a conffile to parse with a list of >=1 LDAP servers to do fallback searches with. + +def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: configparser.ConfigParser): + response = None + + if ldap_config == None: + raise EmptyConfiguration( + "Search Attempted with \"None\" config object." + ) + + for section in ldap_config.sections(): + print(f"Attempting search with server {section}") + try: + server_url = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Server_URL) + search_base = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Search_Base) + search_user = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_User) + authtok_file = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File) + authtok = get_ldap_authtok(authtok_file) + + searcher = LDAPServer(ldap_server=server_url, ldap_user=search_user, ldap_authtok=authtok) + response = searcher.search(search_ou, search_base, search_filter, attrs) + + #If we get a response from one of the servers, we don't need to check the rest + if not response is None: + print(f"Response found for server {section}.") + break + # Perm issue reading token file + except PermissionError as permError: + print(f"Permission Error when attempting search for {section}: {permError}.") + # Problem getting LDAP Response + except LDAPException as ldapError: + print(f"Exception occurred when attempting search for {section}: {ldapError}.") + continue + + if response is None: + raise NoLDAPResponse( + f"No response found via LDAP servers: {[section for section in ldap_config]}." + ) + + return response + + +def get_ldap_groups(config=None): ldap_group_osggids = set() - searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) - response = searcher.search("groups", "(cn=*)", ["gidNumber"]) + + response = do_ldap_fallback_search( + search_ou="groups", + search_filter="(cn=*)", + attrs=["gidNumber"], + ldap_config=config + ) + for group in response: ldap_group_osggids.add(group["attributes"]["gidNumber"]) return ldap_group_osggids -def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): +def get_ldap_active_users_and_groups(filter_group_name=None, config=None): """ Retrieve a dictionary of active users from LDAP, with their group memberships. """ ldap_active_users = dict() filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") - searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) - response = searcher.search("people", filter_str, ["employeeNumber", "isMemberOf"]) + response = do_ldap_fallback_search( + search_ou="people", + search_filter=filter_str, + attrs=["employeeNumber", "isMemberOf"], + ldap_config=config + ) for person in response: ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", []) @@ -278,6 +424,7 @@ def provision_group(gid, provision_target, endpoint, authstr): } return call_api3(POST, path, data, endpoint, authstr) + def provision_group_members(gid, prov_id, endpoint, authstr): data = { "RequestType" : "CoPersonProvisioning", diff --git a/create_project.py b/create_project.py index cc447a1..74462de 100755 --- a/create_project.py +++ b/create_project.py @@ -152,4 +152,3 @@ def main(args): except (RuntimeError, urllib.error.HTTPError) as e: print(e, file=sys.stderr) sys.exit(1) - diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index bccd3ce..40fe8a4 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -11,8 +11,6 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" -LDAP_SERVER = "ldaps://ldap-test.cilogon.org" -LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 CACHE_FILENAME = "COmanage_Projects_cache.txt" CACHE_LIFETIME_HOURS = 0.5 @@ -22,26 +20,26 @@ usage: {SCRIPT} [OPTIONS] OPTIONS: - -u USER[:PASS] specify USER and optionally PASS on command line - -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -s LDAP_SERVER specify LDAP server to read data from - -l LDAP_USER specify LDAP user for reading data from LDAP server - -a ldap_authfile specify path to file to open and read LDAP authtok - -d passfd specify open fd to read PASS - -f passfile specify path to file to open and read PASS - -e ENDPOINT specify REST endpoint - (default = {ENDPOINT}) - -o outfile specify output file (default: write to stdout) - -g filter_group filter users by group name (eg, 'ap1-login') - -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile - -n min_users Specify minimum number of users required to update the output file (default: 100) - -h display this help text + -u USER[:PASS] specify USER and optionally PASS on command line + -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -d passfd specify open fd to read PASS + -f passfile specify path to file to open and read PASS + -e ENDPOINT specify REST endpoint (default = {ENDPOINT}) + -o outfile specify output file (default: write to stdout) + -g filter_group filter users by group name (eg, 'ap1-login') + -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -n min_users Specify minimum number of users required to update the output file (default: 100) + -h display this help text PASS for USER is taken from the first of: 1. -u USER:PASS 2. -d passfd (read from fd) 3. -f passfile (read from file) 4. read from $PASS env var + +{utils.LDAP_CONFIG_USAGE_MESSAGE} + """ def usage(msg=None): @@ -58,10 +56,8 @@ class Options: osg_co_id = OSG_CO_ID outfile = None authstr = None - ldap_server = LDAP_SERVER - ldap_user = LDAP_USER - ldap_authtok = None filtergrp = None + ldap_config = None min_users = 100 # Bail out before updating the file if we have fewer than this many users localmaps = [] @@ -80,7 +76,7 @@ def get_osg_co_groups__map(): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h:n:m:') + ops, args = getopt.getopt(args, 'u:c:l:d:f:g:e:o:h:n:m:') except getopt.GetoptError: usage() @@ -89,15 +85,13 @@ def parse_options(args): passfd = None passfile = None - ldap_authfile = None + ldap_auth_path = None for op, arg in ops: if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) - if op == '-s': options.ldap_server= arg - if op == '-l': options.ldap_user = arg - if op == '-a': ldap_authfile = arg + if op == '-l': ldap_config_path = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg @@ -109,9 +103,13 @@ def parse_options(args): try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) - options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile) except PermissionError: usage("PASS required") + + try: + options.ldap_config = utils.read_ldap_conffile(ldap_config_path) + except utils.EmptyConfiguration: + usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") def _deduplicate_list(items): """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. @@ -120,7 +118,7 @@ def _deduplicate_list(items): return list(dict.fromkeys(items)) def get_osguser_groups(filter_group_name=None): - ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) + ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=options.ldap_config) topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() project_names = topology_projects.keys() diff --git a/project_group_setup.py b/project_group_setup.py index a263e04..dc953d3 100755 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -7,8 +7,6 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry.cilogon.org/registry/" -LDAP_SERVER = "ldaps://ldap.cilogon.org" -LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 7 UNIX_CLUSTER_ID = 1 LDAP_TARGET_ID = 6 @@ -21,25 +19,25 @@ usage: [PASS=...] {SCRIPT} [OPTIONS] OPTIONS: - -u USER[:PASS] specify USER and optionally PASS on command line - -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) - -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) - -s LDAP_SERVER specify LDAP server - -y LDAP_USER specify LDAP server user - -p LDAP authtok specify LDAP server authtok - -d passfd specify open fd to read PASS - -f passfile specify path to file to open and read PASS - -e ENDPOINT specify REST endpoint - (default = {ENDPOINT}) - -o outfile specify output file (default: write to stdout) - -h display this help text + -u USER[:PASS] specify USER and optionally PASS on command line + -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -t LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) + -d passfd specify open fd to read PASS + -f passfile specify path to file to open and read PASS + -e ENDPOINT specify REST endpoint (default = {ENDPOINT}) + -o outfile specify output file (default: write to stdout) + -h display this help text PASS for USER is taken from the first of: 1. -u USER:PASS 2. -d passfd (read from fd) 3. -f passfile (read from file) 4. read from $PASS env var + +{utils.LDAP_CONFIG_USAGE_MESSAGE} + """ @@ -57,12 +55,10 @@ class Options: osg_co_id = OSG_CO_ID ucid = UNIX_CLUSTER_ID provision_target = LDAP_TARGET_ID - ldap_user = LDAP_USER - ldap_server = LDAP_SERVER outfile = None authstr = None - ldap_authtok = None project_gid_startval = PROJECT_GIDS_START + ldap_config = None options = Options() @@ -70,7 +66,7 @@ class Options: def parse_options(args): try: - ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:s:y:h") + ops, args = getopt.getopt(args, "u:c:g:l:t:a:d:f:e:o:h") except getopt.GetoptError: usage() @@ -89,10 +85,10 @@ def parse_options(args): options.osg_co_id = int(arg) if op == "-g": options.ucid = int(arg) - if op == "-l": + if op == "-t": options.provision_target = int(arg) - if op == "-p": - options.ldap_authtok = arg + if op == '-l': + ldap_config_path = arg if op == "-d": passfd = int(arg) if op == "-f": @@ -101,10 +97,6 @@ def parse_options(args): options.endpoint = arg if op == "-o": options.outfile = arg - if op == "-s": - options.ldap_server = arg - if op == "-y": - options.ldap_user = arg try: user, passwd = utils.getpw(options.user, passfd, passfile) @@ -112,6 +104,11 @@ def parse_options(args): except PermissionError: usage("PASS required") + try: + options.ldap_config = utils.read_ldap_conffile(ldap_config_path) + except utils.EmptyConfiguration: + usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") + def append_if_project(project_groups, group): """If this group has a ospoolproject id, and it starts with "Yes-", it's a project""" @@ -182,7 +179,7 @@ def get_projects_needing_cluster_groups(project_groups): def get_projects_needing_provisioning(project_groups): # project groups provisioned in LDAP - ldap_group_osggids = utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok) + ldap_group_osggids = utils.get_ldap_groups(options.ldap_config) try: # All project osggids project_osggids = set( diff --git a/requirements.txt b/requirements.txt index aa2a128..7eb0806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -ldap3~=2.9.1 \ No newline at end of file +ldap3~=2.9.1 +requests>=v2.32.4 +