|     IEEE Xplore Digital Library     |     IEEE Standards     |     IEEE Spectrum     |     More Sites

Verified Commit 067a25bb authored by Emi Simpson's avatar Emi Simpson
Browse files

Cut out (most of) the web interface

parent c84c507e
Pipeline #1118 failed with stage
in 46 seconds
from flask_saml2.utils import certificate_from_file, private_key_from_file
from mystic.auth.saml import SamlAuth
from import DirectAuth
......@@ -20,8 +19,7 @@ PROJECTS_FILE = "/path/to/projects.json"
COORDINATOR_SOCKET = "/path/to/mystic_coordinator.sock" # Unix Socket
#COORDINATOR_SOCKET = ('', 6060) # TCP Socket
#AUTH = SamlAuth()
AUTH = DirectAuth()
# SAML Options
......@@ -63,4 +63,4 @@ very professional CDN][3.5]
from mystic import views #type:ignore
from mystic.init_database import setup_database #type:ignore
from mystic.config import create_app #type:ignore
from mystic.frontend import ErrorTuple, get_project, get_user, get_user_force_login #type:ignore
from mystic.frontend import ErrorTuple #type:ignore
from mystic.database import MalformedId, parse_alphaid, Project
from mystic.outcome import GenericAlert, Outcome, Outcomes, ProjectField, ProjectFieldKind
from mystic.queries import ProjectInfo, Query
from mystic.types import ProjectID, WebRequest, UncheckedPID, UserID
from mystic.sources import fold_sources, SOURCE_PROCESSORS
from mystic import queries, outcome
from typing import Callable, cast, Collection, Mapping, Tuple, TypeAlias
Action: TypeAlias = Callable[[WebRequest], Outcomes | Query[Outcomes, Outcomes]]
AuthedProjectAction: TypeAlias = Callable[[WebRequest, UncheckedPID, UserID], Outcomes | Query[Outcomes, Outcomes]]
force_login: Tuple[outcome.ForceLogin] = (outcome.ForceLogin(),)
def delete_source(req: WebRequest) -> Outcomes | Query[Outcomes, Outcomes]:
Remove a source with the ID in `form[source_id]` from a project the active user owns
# Check that the user is logged in
if req.user is not None:
# If so, try building the query
return queries.MappedQuery(
# Delete the source
queries.DeleteSourceIfOwned(int(req.form['source_id']), req.user),
# If successful, rebuild projects.json
on_success=lambda _: (outcome.RebuildProjectsJson(),),
# Otherwise, just put out an error
on_error=lambda id: (outcome.Error(GenericAlert(), f"The source with id {id} doesn't exist, or you don't have permission to delete it"),)
except ValueError:
# If the `source_id` was malformed, do an error
return (outcome.Error(GenericAlert(), "Source ID is not in a valid integer format"),)
except KeyError:
# If the `source_id` wasn't present, that's also an error
return (outcome.Error(GenericAlert(), 'Must include a \"source_id\" to delete'),)
# If the user wasn't logged in, send them to the login page
return (outcome.ForceLogin(),)
def add_source(req: WebRequest, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
Add a source to a project the user owns
Takes the form fields `source_type` and `source_url`.
Use with :meth:`auth_action()`
# Check for the arguments in the form
source_type = req.form['source_type'].strip()
source_url = req.form['source_url'].strip()
# Error if zero-length
if len(source_url) == 0:
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"Make sure you include a URL for your data source"),)
# Identify which processor corresponds to this type of source
source_processor = SOURCE_PROCESSORS[source_type]
except KeyError:
# And throw a fit if this source doesn't exist
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"Invalid source type specified"),)
except KeyError:
# If the form didn't have all the necessary arguments, be sad about it
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"Missing parameter, make sure both 'type' and 'url' are specified"),)
# Validate that the source is shaped right
source = source_processor(-1, Project(pid), source_type, source_url, False)
validation_error = source.validate_source()
if validation_error is not None:
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
# Then actually start building the query
return queries.BoundQuery(
validate_and_fetch_pid(pid, user), # first we check that the user has permissions
transformation=lambda project: queries.MappedQuery( # if so...
# add a source to the project
queries.AddSourceToProject(project.project_id, source_type, source_url),
on_success=lambda source_id: \
# then rebuild the projects.json file
# and submit a new job for scanning this specific source
[f'p-{pid:x}', f's-{source_id}'],
f'Scan {source.display_name()} {source.get_simplified_form()}',
fold_sources(source.expand_source()))] # (plus all its variants)
if project.draft_owner is None else
# if there was a problem adding the source, turn it into an error
on_error=lambda e: ({
# this shouldn't be possible, because we validated that the project
# existed in the last step
queries.AddSourceError.NonexistantProject: outcome.Error(
ProjectField(pid, ProjectFieldKind.Generic),
"[INTERNAL ERROR] This project does not exist"),
# the data source has already been added
queries.AddSourceError.SourceAlreadyPresent: outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"This data source has already been added to this project"),
def add_owner(req: WebRequest, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
Add another owner to one of the user's projects
Takes the form field `new_owner`
Use with :meth:`auth_action()`
# if the new owner isn't specified
if 'new_owner' not in req.form or req.form['new_owner'] is None or len(req.form['new_owner']) == 0:
# pout if so
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddOwner),
"Please specify an owner to add"),)
# start building the query
return queries.BoundQuery(
validate_pid(pid, user), # first validate that the user owns this project
transformation=lambda project_id: queries.MappedQuery(
# then add the new owner
queries.AddOwnerByUsername(req.form['new_owner'], project_id),
on_success=lambda _: tuple(), # nothing needs to happen if we succeed
on_error=lambda error: ({ # if we fail, pick decipher the error by picking 1
# This shouldn't be possible, since we validated the project already
queries.AddOwnerByUsernameError.NonexistantProject: outcome.Error(
ProjectField(project_id, ProjectFieldKind.AddOwner),
"[INTERNAL ERROR] This project does not exist"),
# The user tried to add an owner who already owned the project
queries.AddOwnerByUsernameError.DuplicateOwner: outcome.Error(
ProjectField(project_id, ProjectFieldKind.AddOwner),
"This user is already an owner of this project"),
# The username the user entered does not exist
queries.AddOwnerByUsernameError.UsernameDNE: outcome.Error(
ProjectField(project_id, ProjectFieldKind.AddOwner),
"This user does not exist or hasn't logged in yet"),
def edit_properties(req: WebRequest, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
Edit the properties of one of the user's projects
Takes the form fields `name`, `desc`, and `slug`, but will produce a user-friendly
error if one is missing
Use with :meth:`auth_action()`
# pull out the required fields
name = req.form['name'].strip()
desc = req.form['desc'].strip()
slug = req.form['slug'].strip()
except KeyError:
# and error if any aren't provided
return (outcome.Error(
'Editing properties requires that the "name", "desc", and "slug" arguments are set'),)
# Create an error for each field that's empty, but don't return them yet
missing_fields: Collection[Outcome] = [
outcome.Error(ProjectField(pid, field), msg)
for (field, value, msg)
in (
(ProjectFieldKind.ProjectName, name, "What should we call your project?"),
(ProjectFieldKind.Description, desc, "Tell us a little bit about your project"),
(ProjectFieldKind.Slug, slug, "Pick a short and sweet name to use in URLs"),)
if len(value) == 0 ]
return queries.BoundQuery(
validate_and_fetch_pid(pid, user), # Check that the user owns the project
transformation=lambda project_info: queries.BoundQuery(
# if so...
# Update the project's properties with the new values
*[f if len(f) > 0 else None for f in (slug, name, desc)]),
# If that succeeds, check whether or not the project was a draft that is
# now finalized
on_success=lambda is_final: is_final and project_info.draft_owner is not None,
# If there was an error, decode it
on_error=lambda error: {
# We already checked that the project exists, so this just means no
# change happened. We can report the errors for missing fields and
# nothing else
queries.UpdateProjectError.NonexistantProjectOrNoChange: missing_fields,
# If the slug entered was already taken, let the user know, along with
# any fields they left empty
queries.UpdateProjectError.DuplicateSlug: [outcome.Error(
ProjectField(pid, ProjectFieldKind.Slug),
"It looks like this URL is already taken!"
)] + missing_fields,
transformation=lambda needs_rescan:
# If updating the project properties succeeded, do one of two things:
# If the project was a draft that was just finalized, we need to scan it
# So get a list of the project's sources
on_success=lambda sources: (
# rebuild the projects.json
# and then request that all those sources be rebuilt
[f's-{source.origin_id:x}' for source in sources] + \
f'Scan project {project_info.display_name}',
for source in sources
for e_source in source.expand_source()]))),
on_error=lambda noreturn: noreturn) # impossible
if needs_rescan else
# If the project wasn't just finalized, we don't need to do anything but
# report the missing fields
def remove_owner(req: WebRequest, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
Remove an owner from one of the user's projects
Takes the form field `user`
Use with :meth:`auth_action()`
# Try to sus out the user's ID from the form
owner = UserID(parse_alphaid(req.form['user']))
except KeyError:
# The form doesn't even have the key
return (outcome.Error(GenericAlert(), "Please include the 'user' field"),)
except MalformedId as e:
# The form had a user listed, but their ID was malformed
return (outcome.Error(GenericAlert(), str(e)),)
# If the ID was retrieved successfully, start describing the query
return queries.BoundQuery(validate_pid(pid, user), # check the user owns this project
transformation=lambda pid: queries.MappedQuery(
# if so, remove the attached owner
queries.RemoveOwner(pid, owner),
on_success=lambda _: tuple(), # if it succeeded, nothing else needs to happen
on_error=lambda _: (outcome.Error(
# otherwise, this indicates that the mentioned account isn't an owner
ProjectField(pid, ProjectFieldKind.Generic),
"That user is not an owner of this project"),)))
def delete_project(_: WebRequest, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
Delete one of the user's projects
Use with :meth:`auth_action()`
return queries.BoundQuery(validate_pid(pid, user), # validate ownership
transformation=lambda pid: queries.MappedQuery(
queries.DeleteProject(pid), # try to delete the project
on_success=lambda _: tuple(), # success - nothing needs to change
# error - should be impossible due to the above validation
on_error=lambda _: (outcome.Error(ProjectField(pid, ProjectFieldKind.Generic),
"[INTERNAL ERROR] This project does not exist"),)))
def set_high_contrast(req: WebRequest) -> Outcomes:
Turns on high contrast mode
return (outcome.AmendSession({'high_contrast': req.form.get("enabled") != 'False'}),)
def logout(_: WebRequest) -> Outcomes:
Equivilent to :meth:`outcome.Logout()`
return (outcome.Logout(),)
def auth_action(action: AuthedProjectAction) -> Action:
A wrapper around actions that take an :class:`UncheckedPID` and a :class:`UserID`
This allows using these actions as standard actions
# unauthed_action is the standard signature for actions
def unauthed_action(req: WebRequest) -> Outcomes | Query[Outcomes, Outcomes]:
if req.user is None:
# If the user isn't logged in, send them to the login page
return force_login
# Otherwise, the user is logged in, and we can try to detect what project
# they're talking about
project_id = try_get_pid(req)
if isinstance(project_id, outcome.Error):
# If that didn't work though, that's an error
return (project_id,)
# Otherwise, we now know the project and the user, and can proceed
return action(req, project_id, req.user)
return unauthed_action
def try_get_pid(req: WebRequest) -> outcome.Error | UncheckedPID:
Attempt to determine the project ID specified in a :class:`Request`
# The happy path: Pull the ID out of the form, parse it, and send it on its way
return UncheckedPID(parse_alphaid(req.form['id']))
except MalformedId as e:
# if the ID was malformed, that's an error
return outcome.Error(GenericAlert(), str(e))
except KeyError:
# If the ID wasn't even specified, that's also an error
return outcome.Error(GenericAlert(), "No project id specified")
def validate_pid(project_id: UncheckedPID, user_id: UserID) -> Query[ProjectID, Tuple[outcome.Error]]:
Shorthand for a query that validates the existance of a PID or produces an :class:`outcome.Error`
return queries.MappedQuery(
# Check that we own the project
queries.ValidateProjectOwned(project_id, user_id),
on_success=lambda pid: pid, # Pass through the ID on a success
# But if we don't own it or it doesn't exist, stop execution with an error
on_error=lambda _: (outcome.Error(GenericAlert(), "Either you don't own this project, or it doesn't exist"
def validate_and_fetch_pid(project_id: UncheckedPID, user_id: UserID) -> Query[ProjectInfo, Tuple[outcome.Error]]:
Equivilent to :meth:`validate_pid()`, but returns more information about the PID
return queries.MappedQuery(
# Check that we own the project
queries.RetreiveProjectInfoIfOwned(project_id, user_id),
on_success=lambda info: info, # Pass through project info if we succeed
# Or else halt execution with an error
on_error=lambda _: (outcome.Error(GenericAlert(), "Either you don't own this project, or it doesn't exist"),))
#class RemoveOwner(Action): pass
#class Edit(Action): pass
#class SetHighContrast(Action): pass
#class DeleteProject(Action): pass
#class Logout(Action): pass
ACTION_NAMES: Mapping[str, Action] = {
"delete_source": delete_source,
"remove_owner": auth_action(remove_owner),
"set_high_contrast": set_high_contrast,
"delete_project": auth_action(delete_project),
"logout": logout,
A mapping from action names to the corresponding actions
def get_preserved_form_data(req: WebRequest) -> Collection[Tuple[str, str]]:
Translate form data to flashable messages, preserving it between requests
if req.form.get('action', None) == 'edit':
return [
(req.form[field], f'data-{req.form["id"]}-{field}')
for field in ('name', 'desc', 'slug')
if field in req.form]
return cast(Collection[Tuple[str, str]], tuple())
def identify_action(req: WebRequest) -> outcome.Error | Action:
Based on the form fields specified in the :class:`Request`, pick an action to run
if 'action' not in req.form:
return outcome.Error(GenericAlert(), "No action specified")
action = req.form['action']
if action == 'edit':
if len(req.form.get('new_owner', '').strip()) != 0:
return auth_action(add_owner)
elif len(req.form.get('source_url', '').strip()) != 0:
return auth_action(add_source)
return auth_action(edit_properties)
elif action not in ACTION_NAMES:
return outcome.Error(GenericAlert(), f'Invalid action specified: "{action}"')
return ACTION_NAMES[action]
......@@ -3,11 +3,11 @@ from pymysql.err import IntegrityError
from werkzeug.exceptions import abort
from werkzeug.utils import redirect
from werkzeug.wrappers.response import Response
from mystic.config import get_auth_module, get_database
from flask.blueprints import Blueprint
from flask.helpers import flash
from flask.templating import render_template
from mystic.database import User
from mystic import config
from typing import Optional, Tuple, Union, cast
from argon2 import PasswordHasher
......@@ -23,11 +23,9 @@ bp = Blueprint("direct_login", __name__, url_prefix="/")
def login() -> Union[str, Response]:
db = get_database()
with db.cursor() as c:
if get_auth_module().check_user(c):
redirect_url = '/api/v1/auth/sso/callback' if 'callback' in session else '/'
return redirect(redirect_url)
if 'id' in session:
redirect_url = '/api/v1/auth/sso/callback' if 'callback' in session else '/'
return redirect(redirect_url)
return render_template("login.html",
login=("Login", "#")
......@@ -90,7 +88,7 @@ def run_signup(redirect_url: str = '/') -> None:
if problem_found:
db = get_database()
db = config.get_database()
c = db.cursor()
uid = randint(0, 2**(8 * 4 - 1) - 1)
......@@ -137,7 +135,7 @@ def run_login(redirect_url: str = '/') -> None:
db = get_database()
db = config.get_database()
c = db.cursor()
if '@' in username:
lookup = User.lookup_user_by_email
from mystic.database import User
from typing import Optional
from pymysql.cursors import Cursor
from mystic.auth import AuthModule, update_user
from mystic.types import UserID
from import Flask
from flask.helpers import url_for
from flask_saml2.sp.sp import ServiceProvider
class MysticServiceProvider(ServiceProvider):
def get_default_login_return_url(self) -> str:
return url_for('main.main')
def get_logout_return_url(self) -> str:
return url_for('main.main')
class SamlAuth(AuthModule):
An authenticator that uses a SAML IDP to identify the user
def __init__(
This doesn't take any specific parameters, but instead expects to be able to find
a set of saml-specific parameters in the Flask config. For more information, see
self.service_provider = MysticServiceProvider()
def setup(self, app: Flask) -> None:
app.register_blueprint(self.service_provider.create_blueprint(), url_prefix='/saml/')
def get_login_url(self) -> str:
return self.service_provider.get_login_url()
def check_user(self, c: Cursor) -> Optional[User]:
if self.service_provider.is_user_logged_in():
user_info = self.service_provider.get_auth_data_in_session().attributes
user_id = int(user_info['uidNumber'])
return update_user(
return None
def check_userp(self) -> Optional[UserID]:
if self.service_provider.is_user_logged_in():
user_info = self.service_provider.get_auth_data_in_session().attributes
return UserID(int(user_info['uidNumber']))
return None
def logout(self) -> None:
Log the user out
There is no guarantee that the user with this ID exists at all. The end result of
this call should be that the user is logged out, and future calls to
:func:`check_user()` return `None` until the user takes action to log in again.
......@@ -21,32 +21,35 @@ from __future__ import annotations
from base64 import b64encode
from hashlib import sha1
import json
from mystic.auth import AuthModule
from mystic.auth import direct
from mystic.error_flagging import flag_errors_daemon
from threading import Thread
from mystic.types import Backend
from mystic.projects import produce_project_report
from mystic.coordination import AvailableBackends, CoordinatorConnection
from mystic.coordination import CoordinatorConnection
from os import path
from elasticsearch.client import Elasticsearch
import asyncio
import pymysql
from pymysql.connections import Connection
from pymysql.cursors import Cursor
from flask import Flask, url_for, current_app, session, g
from flask_cors import CORS
from typing import Any, Collection, Dict, Optional, cast
import uuid
from flask_saml2.sp.sp import ServiceProvider
from typing import Any, Dict, Optional, cast
from pymysql.err import OperationalError
from mystic import views
from mystic.api import v1
from mystic.api.v1 import auth as v1_auth
from mystic.auth.saml import SamlAuth
from mystic.init_database import setup_database
from mystic.sources import SOURCE_PROCESSORS
class MysticServiceProvider(ServiceProvider):
def get_default_login_return_url(self) -> str:
return url_for('main.main')
def get_logout_return_url(self) -> str:
return url_for('main.main')
def is_high_contrast() -> bool:
return cast(bool, session.get('high_contrast', False))
......@@ -129,9 +132,6 @@ def save_projects_json(c: Cursor, path: Optional[str] = None) -> None:
with open(path, 'w') as output_file:
json.dump(report, output_file, indent=2)
def get_auth_module() -> AuthModule:
return cast(AuthModule, current_app.config['AUTH'])
def create_app(test_config: Optional[Dict[str, Any]] = None) -> Flask:
app = Flask(__name__)
......@@ -152,28 +152,13 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Flask:
app.register_error_handler(404, v1.error_404)
app.register_error_handler(405, v1.error_405)
auth_module = cast(AuthModule, app.config['AUTH'])
if isinstance(auth_module, SamlAuth):
app.register_blueprint(v1_auth.AuthModule(True, auth_module.service_provider, False).get_blueprint())
if 'SAML2_IDENTITY_PROVIDERS' in app.config:
app.register_blueprint(v1_auth.AuthModule(True, MysticServiceProvider(), False).get_blueprint())
app.register_blueprint(v1_auth.AuthModule(True, None, True).get_blueprint())
async def get_supported_backends() -> Collection[Backend]:
return await (await connect_coordinator()).evaluate_flow(AvailableBackends())
with app.app_context():
supported_backends =
(source.display_name(), source_key)
for (source_key, source)
if source_key in supported_backends