IEEE.org     |     IEEE Xplore Digital Library     |     IEEE Standards     |     IEEE Spectrum     |     More Sites

Verified Commit 0fb1061a authored by Emi Simpson's avatar Emi Simpson
Browse files

Merge branch 'main' of opensource.ieee.org:rit/mystic

parents 3dd5f287 dafac211
Pipeline #332 passed with stages
in 2 minutes and 40 seconds
......@@ -19,10 +19,12 @@ from hashlib import md5
from mystic.sources import SOURCE_PROCESSORS, Source
from functools import reduce
from typing import List, Optional, Dict, Tuple, Type, Union, cast
from typing import Any, List, Optional, Dict, Tuple, Type, Union, cast
from pymysql.cursors import Cursor
from pymysql.err import IntegrityError
import markdown
import bleach
_alphabet = 'abcdefghijklmnopqrstuvwxyz'
......@@ -367,6 +369,107 @@ class Project:
project.add_owner(c, draft_owner, False)
return project
@staticmethod
def get_all_projects(
c: Cursor,
page: int = 0,
page_size: int = 9,
) -> Tuple[List['Project'], bool]:
"""
Produce an excerpt from the list of all projects
Returns two objects: A list of results, and a boolean indicating if there is a
page after this one, sort of like a `has_next`.
Results are paginated, so the provided value should be the page number. Generally
used for populating the Explore page. If you're looking for a specific project,
prefer to use a specific method.
`page_size` can also be specified to set the number of results on each page. Keep
in mind that the page size will affect where page 1 starts, so don't change it
mid-iteration.
Draft projects will not be returned.
Note that pagination starts at 0, so page 0 is the 1st page, page 1 is the 2nd,
and so on.
"""
c.execute('''
SELECT *
FROM projects
WHERE draft_owner IS NULL
LIMIT %s
OFFSET %s;
''', (page_size + 1, page * page_size))
results: Optional[Tuple[Any, ...]] = c.fetchall()
assert results is not None
return ([
Project.from_record(record)
for record in results[:page_size]
], len(results) > page_size)
@staticmethod
def bulk_populate(c: Cursor, projects: List['Project']):
"""
Completely populates the `owners` and `data_sources` fields
This is much more efficient than using one call per project. In addition, all
fields are filled entirely, meaning every owner in the `owners` list will have all
fields loaded.
"""
project_dict = { p.project_id: p for p in projects }
pids_str = ','.join('%s' for _ in range(len(projects)))
c.execute('''
SELECT owners.project_id, users.*
FROM owners
JOIN users ON owners.user_id = users.user_id
WHERE owners.project_id IN(
''' + pids_str + '''
);
''', list(project_dict.keys()))
print(pids_str, project_dict.keys())
results = cast(Tuple[Tuple[int, int, str, str, str, str]], c.fetchall())
print(results)
for result in results:
pid = result[0]
owner_record = result[1:]
project = project_dict[pid]
owner_to_add = User.from_record(*owner_record)
if project._owners == None:
project._owners = [owner_to_add]
else:
project._owners.append(owner_to_add)
c.execute('''
SELECT *
FROM data_sources
WHERE project_id IN(
''' + pids_str + '''
);
''', list(project_dict.keys()))
results = cast(Tuple[Tuple[int, int, str, str, bool]], c.fetchall())
for result in results:
pid = result[1]
project = project_dict[pid]
if project._data_sources == None:
project._data_sources = sources = []
else:
sources = project._data_sources
sources.append(Source(result[0], project, result[2], result[3], result[4]).specialize())
# Any remaining projects with unpopulated fields must have no owners / no sources
for project in projects:
if project._owners is None:
project._owners = []
if project._data_sources is None:
project._data_sources = []
@staticmethod
def from_record(
sql_record: Tuple[int, str, str, Optional[int]]
......@@ -455,6 +558,42 @@ class Project:
self.load(c)
return self._description #type: ignore
def get_short_description(self, c: Cursor) -> str:
"""
Retrieve first paragraph of the description of the project
This MAY trigger a database query if the project has not yet been loaded
Raises:
sqlite3.OperationalError: The project was initialized with a
cursor that points to a database that was not properly set up
mystic.database.NonexistantId: The project was initialized with an ID
that was not found
"""
full_desc = self.get_description(c)
for line in full_desc.split('\n'):
line = line.strip()
if len(line) > 0 and not line.startswith('#'):
return line
return full_desc
def get_rendered_description(self, c: Cursor) -> str:
raw_desc = self.get_description(c)
sanitary_desc = bleach.clean(raw_desc)
return markdown.markdown(
sanitary_desc,
extensions=[
'codehilite',
'def_list',
'fenced_code',
'footnotes',
'nl2br',
'sane_lists',
'smarty',
'tables',
],
)
def set_description(self, description: str) -> None:
"""
Change the description of a project
......@@ -1074,6 +1213,34 @@ class User:
]
return self._projects
@staticmethod
def from_record(
user_id: int,
username: str,
first_name: str,
last_name: str,
email: str
) -> 'User':
"""
Constructs a user from raw parts
Can be called as `User.from_record(*sql_record)`
A properly formatted SQL record returned by a function like `SELECT * FROM
projects` can be converted into a User object using this method.
No validation whatsoever is performed on the resulting User
Raises:
KeyError: The provided record did not have the required number of fields
"""
user = User(user_id)
user._username = username
user._first_name = first_name
user._last_name = last_name
user._email = email
return user
@property
def loaded(self) -> bool:
"""
......
......@@ -135,6 +135,15 @@ class Source(NamedTuple):
def display_name() -> str:
raise NotImplementedError()
def specialize(self) -> 'Source':
"""
Automatically return this source with the methods specific to its subtype
Uses :data:`SOURCE_PROCESSORS` to lookup the source and return this source as that
type. Returns itself if it fails to find an appropriate subclass
"""
return SOURCE_PROCESSORS.get(self.source_type, Source)(*self) #type: ignore
class Git(Source):
@staticmethod
def display_name() -> str:
......
/*
* This is automatically generated by pygments using the following command:
* pygmentize -S default -f html -a .codehilite > styles.css
*/
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.codehilite .hll { background-color: #ffffcc }
.codehilite { background: #f8f8f8; }
.codehilite .c { color: #408080; font-style: italic } /* Comment */
.codehilite .err { border: 1px solid #FF0000 } /* Error */
.codehilite .k { color: #008000; font-weight: bold } /* Keyword */
.codehilite .o { color: #666666 } /* Operator */
.codehilite .ch { color: #408080; font-style: italic } /* Comment.Hashbang */
.codehilite .cm { color: #408080; font-style: italic } /* Comment.Multiline */
.codehilite .cp { color: #BC7A00 } /* Comment.Preproc */
.codehilite .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */
.codehilite .c1 { color: #408080; font-style: italic } /* Comment.Single */
.codehilite .cs { color: #408080; font-style: italic } /* Comment.Special */
.codehilite .gd { color: #A00000 } /* Generic.Deleted */
.codehilite .ge { font-style: italic } /* Generic.Emph */
.codehilite .gr { color: #FF0000 } /* Generic.Error */
.codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */
.codehilite .gi { color: #00A000 } /* Generic.Inserted */
.codehilite .go { color: #888888 } /* Generic.Output */
.codehilite .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
.codehilite .gs { font-weight: bold } /* Generic.Strong */
.codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
.codehilite .gt { color: #0044DD } /* Generic.Traceback */
.codehilite .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
.codehilite .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
.codehilite .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
.codehilite .kp { color: #008000 } /* Keyword.Pseudo */
.codehilite .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
.codehilite .kt { color: #B00040 } /* Keyword.Type */
.codehilite .m { color: #666666 } /* Literal.Number */
.codehilite .s { color: #BA2121 } /* Literal.String */
.codehilite .na { color: #7D9029 } /* Name.Attribute */
.codehilite .nb { color: #008000 } /* Name.Builtin */
.codehilite .nc { color: #0000FF; font-weight: bold } /* Name.Class */
.codehilite .no { color: #880000 } /* Name.Constant */
.codehilite .nd { color: #AA22FF } /* Name.Decorator */
.codehilite .ni { color: #999999; font-weight: bold } /* Name.Entity */
.codehilite .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
.codehilite .nf { color: #0000FF } /* Name.Function */
.codehilite .nl { color: #A0A000 } /* Name.Label */
.codehilite .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
.codehilite .nt { color: #008000; font-weight: bold } /* Name.Tag */
.codehilite .nv { color: #19177C } /* Name.Variable */
.codehilite .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
.codehilite .w { color: #bbbbbb } /* Text.Whitespace */
.codehilite .mb { color: #666666 } /* Literal.Number.Bin */
.codehilite .mf { color: #666666 } /* Literal.Number.Float */
.codehilite .mh { color: #666666 } /* Literal.Number.Hex */
.codehilite .mi { color: #666666 } /* Literal.Number.Integer */
.codehilite .mo { color: #666666 } /* Literal.Number.Oct */
.codehilite .sa { color: #BA2121 } /* Literal.String.Affix */
.codehilite .sb { color: #BA2121 } /* Literal.String.Backtick */
.codehilite .sc { color: #BA2121 } /* Literal.String.Char */
.codehilite .dl { color: #BA2121 } /* Literal.String.Delimiter */
.codehilite .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
.codehilite .s2 { color: #BA2121 } /* Literal.String.Double */
.codehilite .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
.codehilite .sh { color: #BA2121 } /* Literal.String.Heredoc */
.codehilite .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
.codehilite .sx { color: #008000 } /* Literal.String.Other */
.codehilite .sr { color: #BB6688 } /* Literal.String.Regex */
.codehilite .s1 { color: #BA2121 } /* Literal.String.Single */
.codehilite .ss { color: #19177C } /* Literal.String.Symbol */
.codehilite .bp { color: #008000 } /* Name.Builtin.Pseudo */
.codehilite .fm { color: #0000FF } /* Name.Function.Magic */
.codehilite .vc { color: #19177C } /* Name.Variable.Class */
.codehilite .vg { color: #19177C } /* Name.Variable.Global */
.codehilite .vi { color: #19177C } /* Name.Variable.Instance */
.codehilite .vm { color: #19177C } /* Name.Variable.Magic */
.codehilite .il { color: #666666 } /* Literal.Number.Integer.Long */
......@@ -23,6 +23,7 @@
}
</style>
<link rel=stylesheet href={{static_versioned("styles.css")}}/>
<link rel=stylesheet href={{static_versioned("pygments.css")}}/>
<title>{% block title %}{% endblock %} - Mystic</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
{% endblock %}
......@@ -30,7 +31,7 @@
<body>
<nav>
<a href="/" {% if selected is sameas "home" %} class='selected'{% endif %}>HOME</a>
<a href="/stats/" {% if selected is sameas "stats" %} class='selected'{% endif %}>STATS</a>
<a href="/explore/" {% if selected is sameas "explore" %} class='selected'{% endif %}>EXPLORE</a>
<div class="nav-spacer"></div>
<form method=POST id="contrast-toggle">
{% if high_contrast() %}
......
{% extends "base.html" %}
{% import "utils.html" as utils %}
{% set selected = "explore" %}
{% block title %} Explore {% endblock %}
{% block head %}
{{super()}}
<style>
#projects {
display: flex;
flex-wrap: wrap;
justify-content: center;
gap: 40px;
}
#projects > .project {
background-color: #F2F4F5;
width: min(400px, 80%);
padding: 10px 0;
display: flex;
flex-direction: column;
}
.project-spacer {
flex-grow: 1;
}
.project h6 > a {
color: black;
}
.project > h6 {
margin: 10px 40px;
}
.project > * {
margin-left: 40px;
margin-right: 40px;
}
.owners {
display: grid;
grid-template-columns: 1fr 1fr;
}
.owner {
display: grid;
grid-template-columns: 0fr 1fr;
align-items: center;
gap: 5px;
font-size: 0.8em;
}
.avi {
width: 24px;
border-radius: 100px;
}
.languagebar {
display: flex;
width: 100%;
margin: 0;
}
.language {
border-top: 8px solid black;
text-align: center;
padding-top: 6px;
font-size: 0.8em;
color: #666;
}
.language.gitlab {
border-top-color: purple;
}
.language.git {
border-top-color: cornflowerblue;
}
.language.rss {
border-top-color: orangered;
}
#navbuttons {
display: grid;
grid-template-columns: 0fr 1fr 0fr;
width: calc(100% - 160px);
margin: 30px 80px;
}
</style>
{% endblock %}
{% block main %}
<header>
<h2>Explore</h2>
</header>
<div id=projects>
{% for project in projects %}
<div class=project>
<h6>
<a href="/{{id}}"> {{project.get_display_name(cursor)}}</a>
</h6>
<p class="small">
{{project.get_short_description(cursor)}}
</p>
<div class="project-spacer"></div>
<div class="owners">
{% for owner in project.get_owners(cursor) %}
<div class="owner">
<img class="avi" src={{owner.get_avatar(cursor)}}></img>
<p class="owner-name">
{{ owner.get_first_name(cursor) }}
{{ owner.get_last_name(cursor) }}
</p>
</div>
{% endfor %}
</div>
<div class="languagebar">
{% for source_type, sources in project.get_folded_data_sources(cursor).items() %}
<div
class="language {{sources[0].source_type}}"
style="flex-grow: {{sources|length}}">
{{source_type.display_name()}}
</div>
{% endfor %}
</div>
</div>
{% endfor %}
</div>
<div id=navbuttons>
{% if page > 0 %}
<a href="{{page - 1}}">
<button class="big">Prev</button>
</a>
{% endif %}
{% if has_next_page %}
<a href="{{page + 1}}" style="grid-column-start: 3">
<button class="big">Next</button>
</a>
{% endif %}
</div>
{% endblock %}
......@@ -143,7 +143,7 @@
</div>
</section>
<section>
<p> {{project.get_description(cursor)}} </p>
<p> {{project.get_rendered_description(cursor) | safe}} </p>
</section>
<div id=links-and-jobs>
<section>
......
......@@ -47,7 +47,7 @@ header {
<a href="/{{id}}"> {{project.get_display_name(cursor)}}</a>
</h6>
<p class="small">
{{project.get_description(cursor)}}
{{project.get_short_description(cursor)}}
</p>
{% set sources_empty = project.get_data_sources(cursor)|length == 0 %}
<div class="sources{% if sources_empty %} empty{% endif %}">
......
......@@ -106,6 +106,27 @@ def view_project(pid: str) -> str:
dashboards=dashboards
)
@bp.route("/explore/")
@bp.route("/explore/<int:page>")
def explore_paginated(page: int = 0) -> str:
'''
Render the explore page
'''
db = get_database()
with db.cursor() as cursor:
all_projects, has_next_page = Project.get_all_projects(cursor, page = page)
Project.bulk_populate(cursor, all_projects)
return render_template(
"explore.html",
projects=all_projects,
page=page,
has_next_page=has_next_page,
login=_login_or_profile(cursor),
cursor=cursor
)
@bp.post("/")
def post_main() -> str:
"""
......
......@@ -11,6 +11,9 @@ setup(
'flask-saml2 @ git+https://github.com/Alch-Emi/flask-saml2@timestamps#egg=flask-saml2',
'msgpack==1.0.2',
'msgpack-types',
'argon2-cffi'
'argon2-cffi',
'markdown',
'bleach',
'Pygments',
],
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment