(INF-3541) Comanage LDAP Upgrade + Fixes

Dockerfile

@@ -1,4 +1,4 @@
-FROM library/python:3.7-alpine
+FROM library/python:3.9-alpine
 
 LABEL maintainer OSG Software <help@opensciencegrid.org>
 

comanage_utils.py

@@ -4,34 +4,61 @@
 import re
 import json
 import time
+import configparser
 import urllib.error
 import urllib.request
+from enum import Enum
+from pathlib import Path
 from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls
+from ldap3.core.exceptions import LDAPException
 from dataclasses import dataclass
 
 #PRODUCTION VALUES
 
-PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/"
-PRODUCTION_LDAP_SERVER = "ldaps://ldap.cilogon.org"
-PRODUCTION_LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
-PRODUCTION_OSG_CO_ID = 7
-PRODUCTION_UNIX_CLUSTER_ID = 1
-PRODUCTION_LDAP_TARGET_ID = 6
-LDAP_BASE_DN = "o=OSG,o=CO,dc=cilogon,dc=org"
+# PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/"
+# PRODUCTION_OSG_CO_ID = 7
+# PRODUCTION_UNIX_CLUSTER_ID = 1
+# PRODUCTION_LDAP_TARGET_ID = 6
 
 #TEST VALUES
 
-TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/"
-TEST_LDAP_SERVER = "ldaps://ldap-test.cilogon.org"
-TEST_LDAP_USER ="uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
-TEST_OSG_CO_ID = 8
-TEST_UNIX_CLUSTER_ID = 10
-TEST_LDAP_TARGET_ID = 9
+# TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/"
+# TEST_OSG_CO_ID = 8
+# TEST_UNIX_CLUSTER_ID = 10
+# TEST_LDAP_TARGET_ID = 9
 
 # Value for the base of the exponential backoff
 TIMEOUT_BASE = 5
 MAX_ATTEMPTS = 5
 
+# LDAP Server Connection and Search Config, required keys
+class LDAP_CONFIG_KEYS(str, Enum):
+    LDAP_Server_URL = "LDAPServerURl"
+    LDAP_Search_Base = "SearchBase"
+    LDAP_User = "User"
+    LDAP_AuthTok_File = "AuthTokenFile"
+
+    def __str__(self):
+        return f'{self.value}'
+
+LDAP_CONFIG_USAGE_MESSAGE = f"""
+LDAP CONNECTION CONFIG:
+File at LDAP_CONFIG_PATH should be in ini format, servers will be attempted in descending order.
+An example section of this config file follows:
+
+---
+
+[human_server_name]          # arbitrary human label for this server's config
+{LDAP_CONFIG_KEYS.LDAP_Server_URL} = ldaps://ldap-replica-1.osg.chtc.io      # URL to reach this LDAP server from
+{LDAP_CONFIG_KEYS.LDAP_Search_Base} = dc=osg-htc,dc=org                      # LDAP Search Base 
+{LDAP_CONFIG_KEYS.LDAP_User} = cn=readonly,ou=system,dc=osg-htc,dc=org       # full LDAP user DN 
+{LDAP_CONFIG_KEYS.LDAP_AuthTok_File} = /etc/ldap-secrets/osg-ldap/authtoken  # file containing authtoken for access
+
+---
+
+Config file should contain one such section per LDAP server to communicate with.
+"""
+
 
 GET    = "GET"
 PUT    = "PUT"
@@ -43,11 +70,18 @@ class Error(Exception):
     """Base exception class for all exceptions defined"""
     pass
 
-
 class URLRequestError(Error):
     """Class for exceptions due to not being able to fulfill a URLRequest"""
     pass
 
+class EmptyConfiguration(Error):
+    """Class for exceptions due to loading an empty Config file, or one where every section lacked the required keys"""
+    pass
+
+class NoLDAPResponse(Error):
+    """Class for exceptions due to being unable to get any request from any configured LDAP servers."""
+    pass
+
 
 def getpw(user, passfd, passfile):
     if ":" in user:
@@ -79,6 +113,52 @@ def get_ldap_authtok(ldap_authfile):
     return ldap_authtok
 
 
+def read_ldap_conffile(ldap_conffile_path):
+    config = configparser.ConfigParser(allow_no_value=True)
+
+    print(f"Attempting to read config from {ldap_conffile_path}")
+    config.read(ldap_conffile_path)
+    misconfigured_sections = list()
+    for section in config.sections():
+        for key in LDAP_CONFIG_KEYS:
+            # All servers must have all required keys for operation
+            if not config.has_option(section, key) or config.get(section, key) == "":
+                print(f"Section \"{section}\": required key \"{key}\" missing, ignoring section.")
+                misconfigured_sections.append(section)
+                break
+        # For-Else to only check key values if we know the required ones exist (i.e. we didn't break)
+        else:
+            # All server AuthTok files must exist, be files, and not be empty
+            token_path = Path(config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File))
+            try:
+                if not token_path.exists():
+                    print(f"Section \"{section}\": AuthToken File missing or non-file, ignoring section.")
+                    misconfigured_sections.append(section)
+                    continue
+                if token_path.stat().st_size == 0 :
+                    print(f"Section \"{section}\": AuthToken File is empty, ignoring section.")
+                    misconfigured_sections.append(section)
+                    continue
+            except OSError as e:
+                print(f"Section \"{section}\": Exception raised while checking AuthTok File: {e}, skipping section.")
+                misconfigured_sections.append(section)
+                continue
+
+    for section in misconfigured_sections:
+        print(f"Dropping section {section}")
+        config.remove_section(section)
+
+    # if their are no servers in the file that have all required keys
+    if len(config.sections()) == 0:
+        #when script needs to say LDAP Config required, raise a EmptyConfiguration error
+        #usage("LDAP Config File Required")
+        raise EmptyConfiguration(
+            f"Config file at {ldap_conffile_path} was empty or all sections lacked required keys."
+        )
+    print(f"Finished reading config from {ldap_conffile_path}")
+    return config
+
+
 def mkrequest(method, target, data, endpoint, authstr, **kw):
     url = os.path.join(endpoint, target)
     if kw:
@@ -175,7 +255,7 @@ def get_datalist(data, listname):
     return data[listname] if data else []
 
 
-class LDAPSearch:
+class LDAPServer:
     """ Wrapper class for LDAP searches. """
     server: Server = None
     connection: Connection = None
@@ -184,27 +264,93 @@ def __init__(self, ldap_server, ldap_user, ldap_authtok):
         self.server = Server(ldap_server, get_info=ALL)
         self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
 
-    def search(self, ou, filter_str, attrs):
-        _, _, response, _ = self.connection.search(f"ou={ou},{LDAP_BASE_DN}", filter_str, attributes=attrs)
+    def search(self, ou, search_base, filter_str, attrs):
+        # simple paged search
+        # https://github.com/cannatag/ldap3/blob/7991e67d0a2fb2c1f9cbf832d110ad29fc378f9b/docs/manual/source/standard.rst#L4
+        # https://ldap3.readthedocs.io/en/latest/tutorial_searches.html#simple-paged-search
+        response = self.connection.extend.standard.paged_search(
+            f"ou={ou},{search_base}",
+            filter_str, 
+            attributes=attrs,
+            paged_size=500,
+            generator=True
+        )
+
         return response
 
-def get_ldap_groups(ldap_server, ldap_user, ldap_authtok):
+
+# TODO:
+# do_ldap_fallback_search, get_ldap_groups, and get_ldap_active_users_and_groups should be a method of the LDAPSearch class
+# script calling this lib should init LDAPSearch, then call the method that asks for the info it wants.
+# Be able to feed in either one server's config to the LDAPSearch, or a conffile to parse with a list of >=1 LDAP servers to do fallback searches with.
+
+def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: configparser.ConfigParser):
+    response = None
+
+    if ldap_config == None:
+        raise EmptyConfiguration(
+            "Search Attempted with \"None\" config object."
+        )
+
+    for section in ldap_config.sections():
+        print(f"Attempting search with server {section}")
+        try:
+            server_url = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Server_URL)
+            search_base = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Search_Base)
+            search_user = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_User)
+            authtok_file = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File)
+            authtok = get_ldap_authtok(authtok_file)
+
+            searcher = LDAPServer(ldap_server=server_url, ldap_user=search_user, ldap_authtok=authtok)
+            response = searcher.search(search_ou, search_base, search_filter, attrs)
+
+            #If we get a response from one of the servers, we don't need to check the rest 
+            if not response is None:
+                print(f"Response found for server {section}.")
+                break
+        # Perm issue reading token file
+        except PermissionError as permError:
+            print(f"Permission Error when attempting search for {section}: {permError}.")
+        # Problem getting LDAP Response
+        except LDAPException as ldapError:
+            print(f"Exception occurred when attempting search for {section}: {ldapError}.")
+            continue
+
+    if response is None:
+        raise NoLDAPResponse(
+            f"No response found via LDAP servers: {[section for section in ldap_config]}."
+        )
+
+    return response
+
+
+def get_ldap_groups(config=None):
     ldap_group_osggids = set()
-    searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok)
-    response = searcher.search("groups", "(cn=*)", ["gidNumber"])
+
+    response = do_ldap_fallback_search(
+        search_ou="groups",
+        search_filter="(cn=*)",
+        attrs=["gidNumber"],
+        ldap_config=config
+    )
+
     for group in response:
         ldap_group_osggids.add(group["attributes"]["gidNumber"])
     return ldap_group_osggids
 
 
-def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None):
+def get_ldap_active_users_and_groups(filter_group_name=None, config=None):
     """ Retrieve a dictionary of active users from LDAP, with their group memberships. """
     ldap_active_users = dict()
     filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None 
                   else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
 
-    searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok)
-    response = searcher.search("people", filter_str, ["employeeNumber", "isMemberOf"])
+    response = do_ldap_fallback_search(
+        search_ou="people",
+        search_filter=filter_str,
+        attrs=["employeeNumber", "isMemberOf"],
+        ldap_config=config
+    )
 
     for person in response:
         ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", [])
@@ -278,6 +424,7 @@ def provision_group(gid, provision_target, endpoint, authstr):
     }
     return call_api3(POST, path, data, endpoint, authstr)
 
+
 def provision_group_members(gid, prov_id, endpoint, authstr):
     data = {
         "RequestType" : "CoPersonProvisioning",

create_project.py

@@ -152,4 +152,3 @@ def main(args):
     except (RuntimeError, urllib.error.HTTPError) as e:
         print(e, file=sys.stderr)
         sys.exit(1)
-

osg-comanage-project-usermap.py

@@ -11,8 +11,6 @@
 SCRIPT = os.path.basename(__file__)
 ENDPOINT = "https://registry-test.cilogon.org/registry/"
 TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/"
-LDAP_SERVER = "ldaps://ldap-test.cilogon.org"
-LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
 OSG_CO_ID = 8
 CACHE_FILENAME = "COmanage_Projects_cache.txt"
 CACHE_LIFETIME_HOURS = 0.5
@@ -22,26 +20,26 @@
 usage: {SCRIPT} [OPTIONS]
 
 OPTIONS:
-  -u USER[:PASS]      specify USER and optionally PASS on command line
-  -c OSG_CO_ID        specify OSG CO ID (default = {OSG_CO_ID})
-  -s LDAP_SERVER      specify LDAP server to read data from
-  -l LDAP_USER        specify LDAP user for reading data from LDAP server
-  -a ldap_authfile    specify path to file to open and read LDAP authtok
-  -d passfd           specify open fd to read PASS
-  -f passfile         specify path to file to open and read PASS
-  -e ENDPOINT         specify REST endpoint
-                        (default = {ENDPOINT})
-  -o outfile          specify output file (default: write to stdout)
-  -g filter_group     filter users by group name (eg, 'ap1-login')
-  -m localmaps        specify a comma-delimited list of local HTCondor mapfiles to merge into outfile
-  -n min_users        Specify minimum number of users required to update the output file (default: 100)
-  -h                  display this help text
+  -u USER[:PASS]         specify USER and optionally PASS on command line
+  -c OSG_CO_ID           specify OSG CO ID (default = {OSG_CO_ID})
+  -l LDAP_CONFIG_PATH    specify path to LDAP Config file for fallback-search servers
+  -d passfd              specify open fd to read PASS
+  -f passfile            specify path to file to open and read PASS
+  -e ENDPOINT            specify REST endpoint (default = {ENDPOINT})
+  -o outfile             specify output file (default: write to stdout)
+  -g filter_group        filter users by group name (eg, 'ap1-login')
+  -m localmaps           specify a comma-delimited list of local HTCondor mapfiles to merge into outfile
+  -n min_users           Specify minimum number of users required to update the output file (default: 100)
+  -h                     display this help text
 
 PASS for USER is taken from the first of:
   1. -u USER:PASS
   2. -d passfd (read from fd)
   3. -f passfile (read from file)
   4. read from $PASS env var
+
+{utils.LDAP_CONFIG_USAGE_MESSAGE}
+
 """
 
 def usage(msg=None):
@@ -58,10 +56,8 @@ class Options:
     osg_co_id = OSG_CO_ID
     outfile = None
     authstr = None
-    ldap_server = LDAP_SERVER
-    ldap_user = LDAP_USER
-    ldap_authtok = None
     filtergrp = None
+    ldap_config = None
     min_users = 100 # Bail out before updating the file if we have fewer than this many users
     localmaps = []
 
@@ -80,7 +76,7 @@ def get_osg_co_groups__map():
 
 def parse_options(args):
     try:
-        ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h:n:m:')
+        ops, args = getopt.getopt(args, 'u:c:l:d:f:g:e:o:h:n:m:')
     except getopt.GetoptError:
         usage()
 
@@ -89,15 +85,13 @@ def parse_options(args):
 
     passfd = None
     passfile = None
-    ldap_authfile = None
+    ldap_auth_path = None
 
     for op, arg in ops:
         if op == '-h': usage()
         if op == '-u': options.user       = arg
         if op == '-c': options.osg_co_id  = int(arg)
-        if op == '-s': options.ldap_server= arg
-        if op == '-l': options.ldap_user  = arg
-        if op == '-a': ldap_authfile      = arg
+        if op == '-l': ldap_config_path   = arg
         if op == '-d': passfd             = int(arg)
         if op == '-f': passfile           = arg
         if op == '-e': options.endpoint   = arg
@@ -109,9 +103,13 @@ def parse_options(args):
     try:
         user, passwd = utils.getpw(options.user, passfd, passfile)
         options.authstr = utils.mkauthstr(user, passwd)
-        options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile)
     except PermissionError:
         usage("PASS required")
+
+    try:
+        options.ldap_config = utils.read_ldap_conffile(ldap_config_path)
+    except utils.EmptyConfiguration:
+        usage("LDAP Config File Required. Was empty or lacked a valid server configuration.")
 
 def _deduplicate_list(items):
     """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. 
@@ -120,7 +118,7 @@ def _deduplicate_list(items):
     return list(dict.fromkeys(items))
 
 def get_osguser_groups(filter_group_name=None):
-    ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name)
+    ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=options.ldap_config)
     topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json()
     project_names = topology_projects.keys()