From 74007cbb7e34117013c97b0692924b1f14ea9a46 Mon Sep 17 00:00:00 2001
From: Matthew Hague <matthew.hague@rhul.ac.uk>
Date: Tue, 6 Aug 2024 23:48:57 +0100
Subject: [PATCH] refactor(moodle): change create-groups to a direct http
Python script
No more Selenium chaos. Hopefully just a smooth command line experience.
---
moodle/README.md | 7 +-
moodle/create-groups.py | 249 ++++++++++++++++++++++++++++++++++++++++
moodle/create-groups.rb | 186 ------------------------------
3 files changed, 253 insertions(+), 189 deletions(-)
create mode 100644 moodle/create-groups.py
delete mode 100755 moodle/create-groups.rb
diff --git a/moodle/README.md b/moodle/README.md
index 77880f5..f70d940 100644
--- a/moodle/README.md
+++ b/moodle/README.md
@@ -1,13 +1,14 @@
# Moodle Scripts
-Scripts for getting bulk information out of Moodle that you might find
-useful.
+Scripts for getting bulk information out of Moodle that you might find useful.
+Some make chaotic use of Selenium, other more refined use of direct http
+requests.
* `assignment-select-users.py` -- select a list of users on the
assignments page (useful for e.g. granting bulk extensions)
* `bulk-enrol.py` -- bulk enrol a bunch of students on a module
-* `create-groups.rb` -- ruby script that takes a CSV file of
+* `create-groups.py` -- script that takes a CSV file of
students/groups and creates the groups on Moodle.
* `quiz-get-urls.py` -- make a CSV of all quizzes on a module, for use
with the other quiz scripts here.
diff --git a/moodle/create-groups.py b/moodle/create-groups.py
new file mode 100644
index 0000000..07bce18
--- /dev/null
+++ b/moodle/create-groups.py
@@ -0,0 +1,249 @@
+
+import browser_cookie3
+import csv
+import re
+import requests
+import sys
+
+from bs4 import BeautifulSoup
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+from typing import Dict, Iterable, List, Optional, Set
+
+if len(sys.argv) < 3:
+ print("Usage: python create-groups.py <url> <groups file>")
+ print("where")
+ print(" <url> is the url of the groups page of the Moodle module")
+ print(
+ " <students file> a text file listing students\n"
+ " <groups file> file format is CSV with headings Email and Group\n"
+ "\n"
+ " E.g.\n"
+ "\n"
+ " Email,Something,Group\n"
+ " email,blah,groupname\n"
+ )
+ print("Uses browser_cookie3 to get your Moodle login cookies from FireFox")
+ print("Make sure you have a logged in browser open.")
+ print("Tweak the script if you prefer another browser.")
+ exit()
+
+url = sys.argv[1]
+groups_file = sys.argv[2]
+
+MOODLE_DOMAIN = "moodle.royalholloway.ac.uk"
+MOODLE_SESSION_COOKIE = "MoodleSession"
+
+SESSKEY_RE = re.compile(r'.*"sesskey":"([^"]*)".*', re.MULTILINE)
+SESSKEY_GRP = 1
+
+COURSEID_RE = re.compile(r".*?id=(\d{4})")
+COURSEID_GRP = 1
+
+SELECTOR_ID_RE = re.compile(r'.*"addselect",\s*"([0-9a-f]*)".*')
+SELECTOR_ID_GRP = 1
+
+GROUP_NAME_RE = re.compile(r"(.*) \(\d*\)")
+GROUP_NAME_GRP = 1
+
+CREATE_GROUP_URL = "https://moodle.royalholloway.ac.uk/group/group.php"
+ADD_MEMBERS_URL_FMT \
+ = "https://moodle.royalholloway.ac.uk/group/members.php?group={groupid}"
+SELECTOR_URL \
+ = "https://moodle.royalholloway.ac.uk/user/selector/search.php"
+ADD_TO_GROUP_URL_FMT \
+ = "https://moodle.royalholloway.ac.uk/group/members.php?group={groupid}"
+
+GET_USER_URL = "https://moodle.royalholloway.ac.uk/lib/ajax/service.php?" \
+ "sesskey={sesskey}&info=core_enrol_get_potential_users"
+
+class Moodle:
+ def __init__(self):
+ self.cookies = self._get_moodle_cookies()
+
+ def _get_moodle_cookies(self) -> Dict[str, str]:
+ """Gets current moodle cookies from firefox"""
+ cookie_jar = browser_cookie3.firefox(domain_name=MOODLE_DOMAIN)
+ cookies = {
+ c.name : str(c.value)
+ for c in cookie_jar
+ if c.name == MOODLE_SESSION_COOKIE
+ }
+
+ if len(cookies) == 0:
+ raise Exception(
+ "No",
+ MOODLE_SESSION_COOKIE,
+ "found, please make sure you are logged in."
+ )
+
+ return cookies
+
+ def get(self, url : str) -> requests.Response:
+ return requests.get(url, cookies=self.cookies)
+
+ def post_data(
+ self, url : str, data : Dict[str, str | List[str]]
+ ) -> requests.Response:
+ """Make a post request and return json
+ :param data: a dict for data argument
+ :returns: request result"""
+ r = requests.post(url, data=data, cookies=self.cookies)
+ print(r.request.body)
+ return r
+
+@dataclass
+class GroupsPageInfo:
+ courseid : str = ""
+ sesskey : str = ""
+ # group name to group id on Moodle
+ groups : Dict[str, str] = field(default_factory=dict)
+
+class GroupCreator:
+ def __init__(self, moodle : Moodle, groups_page_url : str):
+ self.moodle = moodle
+ self.groups_page_url = groups_page_url
+ self.groups_info = self._get_groups_info()
+
+ def do_groups(self, groups : Dict[str, Set[str]]):
+ """Create all groups with students
+ :param groups: dict from group name to list of student emails"""
+ self._make_groups(groups.keys())
+ for name, students in groups.items():
+ self._add_students_to_group(name, students)
+
+ def _get_groups_info(self) -> GroupsPageInfo:
+ info = GroupsPageInfo()
+
+ if m := COURSEID_RE.match(self.groups_page_url):
+ info.courseid = m[COURSEID_GRP]
+
+ page = self.moodle.get(self.groups_page_url).text
+ for line in page.split("\n"):
+ if m := SESSKEY_RE.match(line):
+ info.sesskey = m[SESSKEY_GRP]
+
+ soup = BeautifulSoup(page, "html.parser")
+ for opt in soup.select("#groups option"):
+ if m := GROUP_NAME_RE.match(str(opt["title"])):
+ name = m[GROUP_NAME_GRP]
+ info.groups[name] = str(opt["value"])
+
+ return info
+
+ def _create_moodle_group(self, group : str):
+ """Create a new Moodle group with given group name"""
+ self.moodle.post_data(
+ CREATE_GROUP_URL,
+ {
+ "id" : "",
+ "courseid" : self.groups_info.courseid,
+ "sesskey" : self.groups_info.sesskey,
+ "name" : group,
+ "submitbutton" : "Save+changes",
+ "_qf__group_form" : "1",
+ "mform_isexpanded_id_general" : "1"
+ }
+ )
+
+ def _make_groups(self, names : Iterable[str]):
+ """Make sure all groups exist in list"""
+ for name in names:
+ if name not in self.groups_info.groups:
+ self._create_moodle_group(name)
+ # refresh now we have new groups
+ self.groups_info = self._get_groups_info()
+
+ def _add_students_to_group(self, name : str, students : Set[str]):
+ """Group name must exist, add students emails to it"""
+
+ if name not in self.groups_info.groups:
+ print(f"Unknown group {name}, ignoring.")
+ return
+
+ # get cur members
+ gid = self.groups_info.groups[name]
+ url = ADD_MEMBERS_URL_FMT.format(groupid=gid)
+ page_text = self.moodle.get(url).text
+ page = BeautifulSoup(page_text, "html.parser")
+
+ cur_members = [
+ opt.text.upper()
+ for opt in page.select("#removeselect option")
+ ]
+ todo = [
+ email
+ for email in (email.upper() for email in students)
+ if not any(
+ email in cur_member
+ for cur_member in cur_members
+ )
+ ]
+
+ todo_ids = list()
+
+ # get selector id for searching users
+ selector_id = self._get_selector_id(page_text)
+ for email in todo:
+ user_id = self._get_user_id(selector_id, email)
+ if user_id is None:
+ print(f"WARNING: could not resolve {email}, ignoring")
+ todo_ids.append(user_id)
+
+ if len(todo_ids) > 0:
+ url = ADD_TO_GROUP_URL_FMT.format(groupid=gid)
+ self.moodle.post_data(
+ url,
+ {
+ "sesskey" : self.groups_info.sesskey,
+ "addselect[]" : todo_ids,
+ "add" : "%E2%97%84%C2%A0Add"
+ }
+ )
+
+ def _get_selector_id(self, page_text : str) -> str:
+ """Gets id needed for lookup up users from group edit page"""
+ for line in page_text.split("\n"):
+ if m := SELECTOR_ID_RE.match(line):
+ return m[SELECTOR_ID_GRP]
+ return ""
+
+ def _get_user_id(self, selector_id : str, email : str) -> Optional[str]:
+ """Get user id for email, return None if not found or multiple"""
+ results = self.moodle.post_data(
+ SELECTOR_URL,
+ {
+ "selectorid" : selector_id,
+ "sesskey" : self.groups_info.sesskey,
+ "search" : email,
+ "userselector_searchanywhere" : "true"
+ }
+ ).json()["results"]
+ print(results)
+ if len(results) != 1:
+ return None
+ elif len(results[0]["users"]) != 1:
+ return None
+ else:
+ return results[0]["users"][0]["id"]
+
+def load_groups(groups_file : str) -> Dict[str, Set[str]]:
+ groups = defaultdict(set)
+ with open(groups_file) as f:
+ for row in csv.DictReader(f):
+ email = row["Email"].strip()
+ group = row["Group"].strip()
+
+ # bodge to add live to email address for Moodle
+ if "@live.rhul" not in email:
+ email = email.replace("@rhul", "@live.rhul")
+
+ groups[group].add(email)
+ return groups
+
+groups = load_groups(groups_file)
+moodle = Moodle()
+creator = GroupCreator(moodle, url)
+creator.do_groups(groups)
+
diff --git a/moodle/create-groups.rb b/moodle/create-groups.rb
deleted file mode 100755
index 909a1e8..0000000
--- a/moodle/create-groups.rb
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/ruby
-
-require 'csv'
-require 'capybara'
-require 'selenium-webdriver'
-require 'tmpdir'
-
-# probably want to remove the next line unless you want to customise
-# where temp files are stored
-ENV['TMPDIR'] = '/tmp/firefox'
-# set this to whereever your firefox cookies are (if you don't update
-# you'll have to do a full login, which isn't the end of the world)
-USER_COOKIES =
- '/home/matt/.mozilla/firefox/hdnxtunz.default-release/cookies.sqlite'.freeze
-
-# Usage
-# create-groups.rb <groups file csv> <moodle groups page url>
-#
-# Groups file format is CSV with headings "Email" and "Group"
-#
-# Email,Something,Group
-# email,blah,groupname
-#
-# Tries to save time by copying your cookies from your Firefox
-# cookies.sqlite. Edit the USER_COOKIES variable to point to your
-# cookies to enable this. Then if you're logged in on your normal
-# firefox, login should be a single click with this script
-
-if ARGV.length < 2
- puts 'create-groups.rb <groups file csv> <moodle groups page url>'
- exit(-1)
-end
-
-GROUPS_FILE = ARGV[0]
-MODULE_GROUPS_PAGE = ARGV[1]
-
-def make_profile_path
- profile = Selenium::WebDriver::Firefox::Profile.new
- profile_path = profile.layout_on_disk
-
- if File.exist? USER_COOKIES
- profile_cookies_file =
- "#{profile_path}#{File::SEPARATOR}cookies.sqlite"
- FileUtils.copy USER_COOKIES, profile_cookies_file
- end
-
- profile_path
-end
-
-def create_capybara
- Capybara.ignore_hidden_elements = false
- Capybara.default_max_wait_time = 20
-
- Capybara.register_driver :selenium do |app|
- profile_path = make_profile_path
- options = Selenium::WebDriver::Firefox::Options.new
- options.add_argument('-profile')
- options.add_argument(profile_path)
- options.binary = '/var/lib/flatpak/exports/bin/org.mozilla.firefox'
- Capybara::Selenium::Driver.new(app, options: options)
- end
-
- Capybara::Session.new :selenium
-end
-
-# keeps trying
-def safe_first(browser, sel, **args)
- loop do
- return browser.first(sel, **args)
- rescue StandardError => e
- puts "Trying again to find #{sel} with #{args} error #{e}"
- end
-end
-
-# returns nil if not found
-def safe_first_nil(browser, sel, **args)
- browser.first(sel, **args)
-rescue StandardError
- nil
-end
-
-# Login
-def login
- browser = create_capybara
- browser.visit('https://moodle.rhul.ac.uk')
- puts 'Please login and press enter'
- $stdin.gets
- browser
-end
-
-def group_exists(browser, group_name)
- safe_first_nil(
- browser, '#groups option', text: /#{group_name} \(\d*\)/, wait: 1
- )
-end
-
-def create_group(browser, group_name)
- safe_first(browser, '#showcreateorphangroupform').click
- browser.fill_in('Group name', with: group_name)
- browser.find('#id_submitbutton').click
-end
-
-def select_group(browser, group_name)
- # clear any existing selection (ffs)
- browser.all('#groups option').each(&:unselect_option)
- safe_first(
- browser, '#groups option', text: /#{group_name} \(\d*\)/
- ).select_option
-end
-
-def select_create_group(browser, group_name)
- # wait for page to load
- safe_first_nil(browser, '#groups option')
- if !group_exists(browser, group_name)
- create_group(browser, group_name)
- else
- # HACK: need to wait for JS load?
- sleep 2
- select_group(browser, group_name)
- end
-end
-
-def student_exists(browser, student_email)
- safe_first_nil(
- browser, '#removeselect option', text: /.*#{student_email}.*/i, wait: 0
- )
-end
-
-def find_student_for_add(browser, student_email)
- browser.find('#addselect_searchtext').fill_in(with: student_email)
- safe_first_nil(
- browser, '#addselect option', text: /.*#{student_email}.*/i
- )
-end
-
-# Search and add user from add page
-def add_student(browser, student_email)
- return if student_exists(browser, student_email)
-
- student = find_student_for_add(browser, student_email)
-
- if student.nil?
- puts "Could not find #{student_email}, please handle them manually"
- else
- student.click
- browser.find('#add').click
- end
- browser.find('#addselect_clearbutton').click
-end
-
-# returns map from group name to list of emails
-def load_groups(filename)
- groups = {}
- CSV.foreach(filename, headers: true) do |row|
- email = row['Email'].strip
- group = row['Group'].strip
-
- # bodge to add live to email address for Moodle
- email.sub!('@rhul', '@live.rhul') unless email.include? '@live.rhul'
-
- groups[group] = [] unless groups.member? group
-
- groups[group].append(email)
- end
- groups
-end
-
-def do_group(browser, group_name, members)
- select_create_group(browser, group_name)
- browser.find('#showaddmembersform').click
- browser.find('#userselector_autoselectuniqueid').set(false)
- members.each { |member| add_student(browser, member) }
- browser.find('#backcell input').click
-end
-
-def do_groups
- groups = load_groups(GROUPS_FILE)
- browser = login
- browser.visit(MODULE_GROUPS_PAGE)
- groups.each { |group_name, members| do_group(browser, group_name, members) }
-end
-
-do_groups
-
-puts 'When done, press enter.'
-$stdin.gets
--
GitLab