IEEE.org     |     IEEE Xplore Digital Library     |     IEEE Standards     |     IEEE Spectrum     |     More Sites

Commit 1a57ac46 authored by Emi Simpson's avatar Emi Simpson
Browse files

Merge branch 'rearchitect' into 'main'

Trial a new architecture for project editing

See merge request !8
parents ded46205 342ec282
Pipeline #901 passed with stages
in 2 minutes and 45 seconds
......@@ -13,7 +13,8 @@ coverage:
script:
- cat setup.py
- pip3 install -e .
- pip3 install coverage pytest selenium
- pip3 install coverage pytest pyright types-bleach types-Markdown types-PyMySQL
- pyright mystic --warnings
- coverage run --source mystic -m pytest mystic
- coverage report -m --omit "*/tests/*"
build:
......
......@@ -60,7 +60,7 @@ very professional CDN][3.5]
[3]: https://opensource.ieee.org/rit/mystic/-/issues/new?issue "Mystic Issue Tracker"
[3.5]: https://cdn.discordapp.com/attachments/450749288834662410/864626965080047628/callgraph.png
'''
from mystic import views
from mystic.init_database import setup_database
from mystic.config import create_app
from mystic.frontend import try_action, ErrorTuple, get_project, get_user, get_user_force_login
from mystic import views #type:ignore
from mystic.init_database import setup_database #type:ignore
from mystic.config import create_app #type:ignore
from mystic.frontend import ErrorTuple, get_project, get_user, get_user_force_login #type:ignore
from mystic.database import compute_alphaid, MalformedId, parse_alphaid, Project
from mystic.outcome import Backend, GenericAlert, Outcome, Outcomes, ProjectField, ProjectFieldKind, Url
from mystic.queries import ProjectInfo, Query
from mystic.request import ProjectID, Request, UncheckedPID, UserID
from mystic.sources import Source, SOURCE_PROCESSORS
from mystic import queries, outcome
from functools import reduce
from typing import Callable, cast, Collection, Iterable, List, Mapping, Tuple, TypeAlias
Action: TypeAlias = Callable[[Request], Outcomes | Query[Outcomes, Outcomes]]
AuthedProjectAction: TypeAlias = Callable[[Request, UncheckedPID, UserID], Outcomes | Query[Outcomes, Outcomes]]
force_login: Tuple[outcome.ForceLogin] = (outcome.ForceLogin(),)
def _fold_sources(sources: Iterable[Source]) -> Mapping[Backend, Collection[Url]]:
"""
Group :class:`Source`'s :class:`Url`s by their :class:`Backend`
This is most useful for when formatting a list of sources to be fed to the
coordinator, which requires that sources be provided in this format (i.e. a backend
name -> url mapping).
## Example
```python
assert _fold_sources([
Source('git', 'https://my.git.forge/MyUser/MyProject1'),
Source('git', 'https://my.git.forge/MyUser/MyProject2'),
Source('rss', 'https://my.rss.feed/feed.rss')
]) == {
Backend('git'): [
Url('https://my.git.forge/MyUser/MyProject1'),
Url('https://my.git.forge/MyUser/MyProject2')
],
Backend('rss'): [ Url('https://my.rss.feed/feed.rss') ]
}
```
"""
return reduce(
lambda dict, source:
# Append the source to the dict as a backend: [url] pair, merging lists where
# needed
dict | {
Backend(source.source_type):\
dict.get(Backend(source.source_type), []) +\
[Url(source.url)] },
sources,
cast(Mapping[Backend, List[Url]], dict())) # An empty dict + type annotations
def delete_source(req: Request) -> Outcomes | Query[Outcomes, Outcomes]:
"""
Remove a source with the ID in `form[source_id]` from a project the active user owns
"""
# Check that the user is logged in
if req.user is not None:
try:
# If so, try building the query
return queries.MappedQuery(
# Delete the source
queries.DeleteSourceIfOwned(int(req.form['source_id']), req.user),
# If successful, rebuild projects.json
on_success=lambda _: (outcome.RebuildProjectsJson(),),
# Otherwise, just put out an error
on_error=lambda id: (outcome.Error(GenericAlert(), f"The source with id {id} doesn't exist, or you don't have permission to delete it"),)
)
except ValueError:
# If the `source_id` was malformed, do an error
return (outcome.Error(GenericAlert(), "Source ID is not in a valid integer format"),)
except KeyError:
# If the `source_id` wasn't present, that's also an error
return (outcome.Error(GenericAlert(), 'Must include a \"source_id\" to delete'),)
else:
# If the user wasn't logged in, send them to the login page
return (outcome.ForceLogin(),)
def add_source(req: Request, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
"""
Add a source to a project the user owns
Takes the form fields `source_type` and `source_url`.
Use with :meth:`auth_action()`
"""
try:
# Check for the arguments in the form
source_type = req.form['source_type'].strip()
source_url = req.form['source_url'].strip()
# Error if zero-length
if len(source_url) == 0:
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"Make sure you include a URL for your data source"),)
else:
try:
# Identify which processor corresponds to this type of source
source_processor = SOURCE_PROCESSORS[source_type]
except KeyError:
# And throw a fit if this source doesn't exist
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"Invalid source type specified"),)
except KeyError:
# If the form didn't have all the necessary arguments, be sad about it
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"Missing parameter, make sure both 'type' and 'url' are specified"),)
# Validate that the source is shaped right
source = source_processor(-1, Project(pid), source_type, source_url, False)
validation_error = source.validate_source()
if validation_error is not None:
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
validation_error
),)
# Then actually start building the query
return queries.BoundQuery(
validate_and_fetch_pid(pid, user), # first we check that the user has permissions
transformation=lambda project: queries.MappedQuery( # if so...
# add a source to the project
queries.AddSourceToProject(project.project_id, source_type, source_url),
on_success=lambda source_id: \
# then rebuild the projects.json file
[outcome.RebuildProjectsJson()
# and submit a new job for scanning this specific source
,outcome.SubmitCoordinatorJob(
[compute_alphaid(pid), f's-{source_id}'],
f'Scan {source.display_name()} {source.get_simplified_form()}',
20,
_fold_sources(source.expand_source()))] # (plus all its variants)
if project.draft_owner is None else
[],
# if there was a problem adding the source, turn it into an error
on_error=lambda e: ({
# this shouldn't be possible, because we validated that the project
# existed in the last step
queries.AddSourceError.NonexistantProject: outcome.Error(
ProjectField(pid, ProjectFieldKind.Generic),
"[INTERNAL ERROR] This project does not exist"),
# the data source has already been added
queries.AddSourceError.SourceAlreadyPresent: outcome.Error(
ProjectField(pid, ProjectFieldKind.AddSource),
"This data source has already been added to this project"),
}[e],)))
def add_owner(req: Request, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
"""
Add another owner to one of the user's projects
Takes the form field `new_owner`
Use with :meth:`auth_action()`
"""
# if the new owner isn't specified
if 'new_owner' not in req.form or req.form['new_owner'] is None or len(req.form['new_owner']) == 0:
# pout if so
return (outcome.Error(
ProjectField(pid, ProjectFieldKind.AddOwner),
"Please specify an owner to add"),)
# start building the query
return queries.BoundQuery(
validate_pid(pid, user), # first validate that the user owns this project
transformation=lambda project_id: queries.MappedQuery(
# then add the new owner
queries.AddOwnerByUsername(req.form['new_owner'], project_id),
on_success=lambda _: tuple(), # nothing needs to happen if we succeed
on_error=lambda error: ({ # if we fail, pick decipher the error by picking 1
# This shouldn't be possible, since we validated the project already
queries.AddOwnerError.NonexistantProject: outcome.Error(
ProjectField(project_id, ProjectFieldKind.AddOwner),
"[INTERNAL ERROR] This project does not exist"),
# The user tried to add an owner who already owned the project
queries.AddOwnerError.DuplicateOwner: outcome.Error(
ProjectField(project_id, ProjectFieldKind.AddOwner),
"This user is already an owner of this project"),
# The username the user entered does not exist
queries.AddOwnerError.UsernameDNE: outcome.Error(
ProjectField(project_id, ProjectFieldKind.AddOwner),
"This user does not exist or hasn't logged in yet"),
}[error],)))
def edit_properties(req: Request, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
"""
Edit the properties of one of the user's projects
Takes the form fields `name`, `desc`, and `slug`, but will produce a user-friendly
error if one is missing
Use with :meth:`auth_action()`
"""
try:
# pull out the required fields
name = req.form['name'].strip()
desc = req.form['desc'].strip()
slug = req.form['slug'].strip()
except KeyError:
# and error if any aren't provided
return (outcome.Error(
GenericAlert(),
'Editing properties requires that the "name", "desc", and "slug" arguments are set'),)
# Create an error for each field that's empty, but don't return them yet
missing_fields: Collection[Outcome] = [
outcome.Error(ProjectField(pid, field), msg)
for (field, value, msg)
in (
(ProjectFieldKind.ProjectName, name, "What should we call your project?"),
(ProjectFieldKind.Description, desc, "Tell us a little bit about your project"),
(ProjectFieldKind.Slug, slug, "Pick a short and sweet name to use in URLs"),)
if len(value) == 0 ]
return queries.BoundQuery(
validate_and_fetch_pid(pid, user), # Check that the user owns the project
transformation=lambda project_info: queries.BoundQuery(
# if so...
queries.MappedQuery(
# Update the project's properties with the new values
queries.UpdateProjectProperties(
project_info.project_id,
*[f if len(f) > 0 else None for f in (slug, name, desc)]),
# If that succeeds, check whether or not the project was a draft that is
# now finalized
on_success=lambda is_final: is_final and project_info.draft_owner is not None,
# If there was an error, decode it
on_error=lambda error: {
# We already checked that the project exists, so this just means no
# change happened. We can report the errors for missing fields and
# nothing else
queries.UpdateProjectError.NonexistantProjectOrNoChange: missing_fields,
# If the slug entered was already taken, let the user know, along with
# any fields they left empty
queries.UpdateProjectError.DuplicateSlug: [outcome.Error(
ProjectField(pid, ProjectFieldKind.Slug),
"It looks like this URL is already taken!"
)] + missing_fields,
}[error]),
transformation=lambda needs_rescan:
# If updating the project properties succeeded, do one of two things:
# If the project was a draft that was just finalized, we need to scan it
queries.MappedQuery(
# So get a list of the project's sources
queries.GetSources(project_info.project_id),
on_success=lambda sources: (
# rebuild the projects.json
outcome.RebuildProjectsJson(),
# and then request that all those sources be rebuilt
outcome.SubmitCoordinatorJob(
[f's-{source.origin_id:x}' for source in sources] + \
[compute_alphaid(project_info.project_id)],
f'Scan project {project_info.display_name}',
20,
_fold_sources([
e_source
for source in sources
for e_source in source.expand_source()]))),
on_error=lambda noreturn: noreturn) # impossible
if needs_rescan else
# If the project wasn't just finalized, we don't need to do anything but
# report the missing fields
queries.Noop(missing_fields)))
def remove_owner(req: Request, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
"""
Remove an owner from one of the user's projects
Takes the form field `user`
Use with :meth:`auth_action()`
"""
try:
# Try to sus out the user's ID from the form
owner = UserID(parse_alphaid(req.form['user']))
except KeyError:
# The form doesn't even have the key
return (outcome.Error(GenericAlert(), "Please include the 'user' field"),)
except MalformedId as e:
# The form had a user listed, but their ID was malformed
return (outcome.Error(GenericAlert(), str(e)),)
# If the ID was retrieved successfully, start describing the query
return queries.BoundQuery(validate_pid(pid, user), # check the user owns this project
transformation=lambda pid: queries.MappedQuery(
# if so, remove the attached owner
queries.RemoveOwner(pid, owner),
on_success=lambda _: tuple(), # if it succeeded, nothing else needs to happen
on_error=lambda _: (outcome.Error(
# otherwise, this indicates that the mentioned account isn't an owner
ProjectField(pid, ProjectFieldKind.Generic),
"That user is not an owner of this project"),)))
def delete_project(_: Request, pid: UncheckedPID, user: UserID) -> Outcomes | Query[Outcomes, Outcomes]:
"""
Delete one of the user's projects
Use with :meth:`auth_action()`
"""
return queries.BoundQuery(validate_pid(pid, user), # validate ownership
transformation=lambda pid: queries.MappedQuery(
queries.DeleteProject(pid), # try to delete the project
on_success=lambda _: tuple(), # success - nothing needs to change
# error - should be impossible due to the above validation
on_error=lambda _: (outcome.Error(ProjectField(pid, ProjectFieldKind.Generic),
"[INTERNAL ERROR] This project does not exist"),)))
def set_high_contrast(req: Request) -> Outcomes:
"""
Turns on high contrast mode
"""
return (outcome.AmendSession({'high_contrast': req.form.get("enabled") != 'False'}),)
def logout(_: Request) -> Outcomes:
"""
Equivilent to :meth:`outcome.Logout()`
"""
return (outcome.Logout(),)
def auth_action(action: AuthedProjectAction) -> Action:
"""
A wrapper around actions that take an :class:`UncheckedPID` and a :class:`UserID`
This allows using these actions as standard actions
"""
# unauthed_action is the standard signature for actions
def unauthed_action(req: Request) -> Outcomes | Query[Outcomes, Outcomes]:
if req.user is None:
# If the user isn't logged in, send them to the login page
return force_login
else:
# Otherwise, the user is logged in, and we can try to detect what project
# they're talking about
project_id = try_get_pid(req)
if isinstance(project_id, outcome.Error):
# If that didn't work though, that's an error
return (project_id,)
else:
# Otherwise, we now know the project and the user, and can proceed
return action(req, project_id, req.user)
return unauthed_action
def try_get_pid(req: Request) -> outcome.Error | UncheckedPID:
"""
Attempt to determine the project ID specified in a :class:`Request`
"""
try:
# The happy path: Pull the ID out of the form, parse it, and send it on its way
return UncheckedPID(parse_alphaid(req.form['id']))
except MalformedId as e:
# if the ID was malformed, that's an error
return outcome.Error(GenericAlert(), str(e))
except KeyError:
# If the ID wasn't even specified, that's also an error
return outcome.Error(GenericAlert(), "No project id specified")
def validate_pid(project_id: UncheckedPID, user_id: UserID) -> Query[ProjectID, Tuple[outcome.Error]]:
"""
Shorthand for a query that validates the existance of a PID or produces an :class:`outcome.Error`
"""
return queries.MappedQuery(
# Check that we own the project
queries.ValidateProjectOwned(project_id, user_id),
on_success=lambda pid: pid, # Pass through the ID on a success
# But if we don't own it or it doesn't exist, stop execution with an error
on_error=lambda _: (outcome.Error(GenericAlert(), "Either you don't own this project, or it doesn't exist"
),)
)
def validate_and_fetch_pid(project_id: UncheckedPID, user_id: UserID) -> Query[ProjectInfo, Tuple[outcome.Error]]:
"""
Equivilent to :meth:`validate_pid()`, but returns more information about the PID
"""
return queries.MappedQuery(
# Check that we own the project
queries.RetreiveProjectInfoIfOwned(project_id, user_id),
on_success=lambda info: info, # Pass through project info if we succeed
# Or else halt execution with an error
on_error=lambda _: (outcome.Error(GenericAlert(), "Either you don't own this project, or it doesn't exist"),))
#class RemoveOwner(Action): pass
#class Edit(Action): pass
#class SetHighContrast(Action): pass
#class DeleteProject(Action): pass
#class Logout(Action): pass
ACTION_NAMES: Mapping[str, Action] = {
"delete_source": delete_source,
"remove_owner": auth_action(remove_owner),
"set_high_contrast": set_high_contrast,
"delete_project": auth_action(delete_project),
"logout": logout,
}
"""
A mapping from action names to the corresponding actions
"""
def get_preserved_form_data(req: Request) -> Collection[Tuple[str, str]]:
"""
Translate form data to flashable messages, preserving it between requests
"""
if req.form.get('action', None) == 'edit':
return [
(req.form[field], f'data-{req.form["id"]}-{field}')
for field in ('name', 'desc', 'slug')
if field in req.form]
else:
return cast(Collection[Tuple[str, str]], tuple())
def identify_action(req: Request) -> outcome.Error | Action:
"""
Based on the form fields specified in the :class:`Request`, pick an action to run
"""
if 'action' not in req.form:
return outcome.Error(GenericAlert(), "No action specified")
action = req.form['action']
if action == 'edit':
if len(req.form.get('new_owner', '').strip()) != 0:
return auth_action(add_owner)
elif len(req.form.get('source_url', '').strip()) != 0:
return auth_action(add_source)
else:
return auth_action(edit_properties)
elif action not in ACTION_NAMES:
return outcome.Error(GenericAlert(), f'Invalid action specified: "{action}"')
else:
return ACTION_NAMES[action]
......@@ -9,7 +9,7 @@ from abc import ABC
from hashlib import sha1
import json
from mystic.analytics.queries import CommitsByAuthor, CommitsByDate, IssuesByOpenTime, OsfActivityByDate, QueryMachine
from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict, Union
from typing import Any, Dict, List, Optional, Set, TypedDict
from pymysql.cursors import Cursor
from mystic.database import Project
......
......@@ -13,8 +13,6 @@ might be some dashboards which re-use queries
from abc import ABC
from typing import Any, Dict, Generic, List, Type, TypeVar, TypedDict
from flask import json
from elasticsearch.client import Elasticsearch
......@@ -203,7 +201,6 @@ class IssuesByOpenTime(Query[List[str], Dict[float, int]]):
"""
@staticmethod
def perform_query(es: Elasticsearch, args: List[str]) -> Dict[float, int]:
project_name = args
DEGREE: int = 2
results: _QueryResult = es.search(index="github_issues,gitlab", body = {
'size': 3,
......@@ -262,7 +259,6 @@ class OsfActivityByDate(Query[List[str], Dict[str, int]]):
"""
@staticmethod
def perform_query(es: Elasticsearch, args: List[str]) -> Dict[str, int]:
project_name = args
results: _QueryResult = es.search(index="osf_nodes_demo_enriched", body = {
'size': 0,
"query": {
......
......@@ -72,6 +72,21 @@ class AuthModule(ABC):
"""
raise NotImplementedError
from mystic.request import UserID
def check_userp(self) -> Optional[UserID]:
"""
Check if a user is logged in
This is called on most pages. If using an auth module that uses callback URLs to
pass information to the server, then this method should be used to check for the
presence of this information.
If this information is found, then the :func:`update_user` should be
called. Finally if a user was discovered, or an existing session was found in the
cache, that user should be returned, or `None` otherwise.
"""
raise NotImplementedError
def force_login(self, c: Cursor) -> User:
"""
Redirect the user to the login page if not logged in
......
......@@ -12,7 +12,8 @@ from typing import Optional, Tuple, Union, cast
from argon2 import PasswordHasher
from pymysql.cursors import Cursor
from mystic.auth import AuthModule, update_user
from mystic.auth import AuthModule
from mystic.request import UserID
from flask import session, request, url_for
from flask.app import Flask
from random import randint
......@@ -151,8 +152,10 @@ def run_login() -> None:
(user.user_id,)
)
assert num_res > 0, "User exists, but no password was found"
fetched = c.fetchone()
assert fetched is not None
password_hash: str = c.fetchone()[0]
password_hash: str = fetched[0]
c.close()
try:
password_hasher.verify(password_hash, password)
......@@ -209,5 +212,10 @@ class DirectAuth(AuthModule):
return User(cast(int, session['id']))
return None
def check_userp(self) -> Optional[UserID]:
if 'id' in session:
return UserID(cast(int, session['id']))
return None
def logout(self) -> None:
del session['id']
......@@ -3,6 +3,7 @@ from typing import Optional
from pymysql.cursors import Cursor
from mystic.auth import AuthModule, update_user
from mystic.request import UserID
from flask.app import Flask
from flask.helpers import url_for
from flask_saml2.sp.sp import ServiceProvider
......@@ -50,6 +51,13 @@ class SamlAuth(AuthModule):
else:
return None
def check_userp(self) -> Optional[UserID]:
if self.service_provider.is_user_logged_in():
user_info = self.service_provider.get_auth_data_in_session().attributes
return UserID(int(user_info['uidNumber']))
else:
return None
def logout(self) -> None:
"""
Log the user out
......
......@@ -26,7 +26,7 @@ from mystic.error_flagging import flag_errors_daemon
from threading import Thread
from mystic.projects import produce_project_report
from mystic.coordination import CoordinatorConnection
from os import path, getcwd
from os import path
from elasticsearch.client import Elasticsearch
import asyncio
......@@ -34,7 +34,6 @@ import pymysql
from pymysql.connections import Connection
from pymysql.cursors import Cursor
from flask import Flask, url_for, current_app, session, g
from flask.wrappers import Response
from typing import Any, Dict, List, Optional, cast
import uuid
......@@ -192,11 +191,11 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Flask:
Thread(target=run_daemon).start()
app.before_first_request(set_up_error_flagging)
def close_database(_: Optional[BaseException]) -> Response:
def close_database(_: Optional[BaseException]) -> None:
db = getattr(g, 'database', None)
if db is not None:
db.close()
return None #type: ignore
return None
app.teardown_appcontext(close_database)
return app
from asyncio.streams import StreamReader, StreamWriter
from pymysql.cursors import Cursor
from mystic.sources import Source
from typing import Any, AsyncGenerator, Callable, Dict, List, NamedTuple, Optional, Set, Tuple, TypeVar, TypedDict, Union
from typing import Any, AsyncGenerator, Callable, Collection, Dict, List, Mapping, NamedTuple, Optional, Set, Tuple, TypeVar, TypedDict, Union
from mystic.database import Project
from mystic.outcome import Backend, Url
from asyncio import BoundedSemaphore, open_connection, open_unix_connection
import msgpack
......@@ -48,10 +49,10 @@ class CoordinatorConnection:
async def post_job(
self,
tags: List[str],
tags: Collection[str],
desc: str,