IEEE.org     |     IEEE Xplore Digital Library     |     IEEE Standards     |     IEEE Spectrum     |     More Sites

Verified Commit f12aee5b authored by Emi Simpson's avatar Emi Simpson
Browse files

Added flagged sources to the UI

parent 3419e43a
Pipeline #236 failed with stages
in 30 seconds
......@@ -211,7 +211,7 @@ class Project:
raise e
else:
raise NonexistantId("project", self._project_id)
source = SOURCE_PROCESSORS.get(source_type, Source)(c.lastrowid, self, source_type, source_url)
source = SOURCE_PROCESSORS.get(source_type, Source)(c.lastrowid, self, source_type, source_url, False)
if self._data_sources is not None:
self._data_sources.append(source)
return source
......@@ -563,25 +563,27 @@ class Project:
source_id,
projects.*,
data_type,
data_url
data_url,
flagged
FROM data_sources
JOIN projects
ON projects.project_id = data_sources.project_id
WHERE data_sources.project_id = %s
''', (self._project_id,))
records = cast(Tuple[Tuple[int, int, str, str, Optional[int], str, str]], c.fetchall())
records = cast(Tuple[Tuple[int, int, str, str, Optional[int], str, str, bool]], c.fetchall())
source_tuples = (
Source(
record[0], # source_id
Project.from_record(record[1:-2]),
record[-2], # data_type
record[-1], # data_url
Project.from_record(record[1:-3]),
record[-3], # data_type
record[-2], # data_url
record[-1], # flagged
)
for record in records
)
self._data_sources = [
SOURCE_PROCESSORS[source_type](source_id, project, source_type, url)
for (source_id, project, source_type, url)
SOURCE_PROCESSORS[source_type](source_id, project, source_type, url, flagged)
for (source_id, project, source_type, url, flagged)
in source_tuples
]
return self._data_sources
......
......@@ -147,7 +147,7 @@ def subaction_add_source(cursor: Cursor) -> None:
f"add_source",
"Invalid source type specified"
)
source = SOURCE_PROCESSORS[source_type](-1, project, source_type, source_url)
source = SOURCE_PROCESSORS[source_type](-1, project, source_type, source_url, False)
validation_error = source.validate_source()
if validation_error is not None:
......
......@@ -18,7 +18,8 @@ def produce_project_report(c: Cursor) -> Dict[str, Dict[str, List[str]]]:
data_sources.source_id,
projects.*,
data_sources.data_type,
data_sources.data_url
data_sources.data_url,
data_sources.flagged
FROM data_sources
JOIN projects
ON projects.project_id = data_sources.project_id
......@@ -41,7 +42,7 @@ def produce_project_report(c: Cursor) -> Dict[str, Dict[str, List[str]]]:
return d
records = (
Source(r[0], Project.from_record(cast(Tuple[int, str, str, Optional[int]], r[1:-2])), r[-2], r[-1])
Source(r[0], Project.from_record(cast(Tuple[int, str, str, Optional[int]], r[1:-3])), r[-3], r[-2], r[-1])
for r in c.fetchall()
)
......@@ -50,7 +51,9 @@ def produce_project_report(c: Cursor) -> Dict[str, Dict[str, List[str]]]:
for source_tuple
in records
for expanded_source_tuple
in SOURCE_PROCESSORS[source_tuple.source_type](source_tuple.origin_id, source_tuple.project, source_tuple.source_type, source_tuple.url).expand_source()
in SOURCE_PROCESSORS[source_tuple.source_type](source_tuple.origin_id,
source_tuple.project, source_tuple.source_type, source_tuple.url,
source_tuple.flagged).expand_source()
)
report: Dict[str, Dict[str, List[str]]] = {}
......
......@@ -75,6 +75,16 @@ class Source(NamedTuple):
The URL of the resource to be scraped by this source
"""
flagged: bool
"""
Whether or not this source has been flagged as errored
If an error occurs while processing this source, this flag will be set in order to
indicate that the source was errored. This prevents the source from being listed in
the projects.json reports, and should also produce a visual indicator for the user.
This can only be removed by deleting and recreating the source.
"""
def expand_source(self) -> List['Source']:
"""
Given a source, expand it to any component sources
......@@ -135,8 +145,8 @@ class GitHub(Source):
def expand_source(self) -> List[Source]:
return [
Source(self.origin_id, self.project, self.source_type, self.url),
] + Git(self.origin_id, self.project, 'git', self.url.rstrip('/') + '.git').expand_source()
Source(self.origin_id, self.project, self.source_type, self.url, self.flagged),
] + Git(self.origin_id, self.project, 'git', self.url.rstrip('/') + '.git', self.flagged).expand_source()
#TODO GitHub2 analytics (https://github.com/chaoss/grimoirelab-sirmordred#github2-)
def validate_source(self) -> Optional[str]:
......@@ -159,9 +169,9 @@ class GitLab(Source):
URL_REGEX = re.compile(r'https://gitlab\.com/[\w\-]+/[\w\-]+/?')
def expand_source(self) -> List[Source]:
return [
Source(self.origin_id, self.project, self.source_type + ':issue', self.url),
Source(self.origin_id, self.project, self.source_type + ':merge', self.url),
] + Git(self.origin_id, self.project, 'git', self.url.rstrip('/') + '.git').expand_source()
Source(self.origin_id, self.project, self.source_type + ':issue', self.url, self.flagged),
Source(self.origin_id, self.project, self.source_type + ':merge', self.url, self.flagged),
] + Git(self.origin_id, self.project, 'git', self.url.rstrip('/') + '.git', self.flagged).expand_source()
def validate_source(self) -> Optional[str]:
if GitLab.URL_REGEX.match(self.url) is None:
......
......@@ -104,6 +104,12 @@
color: var(--empty-grey-box-text);
font-size: 1.3rem;
}
.edit-data-sources .delete-and-warning-container {
display: flex;
justify-content: end;
align-items: end;
color: #e60;
}
.edit-owners {
display: flex;
......
......@@ -54,14 +54,19 @@
<span style="grid-row-end: span {{sources|length}}">{{source_type.display_name()}}</span>
{% for source in sources %}
<a href="{{source.url}}">{{source.url}}</a>
<form class=delete method="POST">
<input type="hidden" name="action" value="delete_source"/>
<input type="hidden" name="source_id" value="{{source.origin_id}}"/>
<input type="hidden" name="id" value="{{id}}"/>
<button class="icon-button">
{{ utils.show_icon("trash-2", "delete source", 16) }}
</button>
</form>
<div class="delete-and-warning-container">
{% if source.flagged %}
{{utils.show_icon("alert-triangle", "problems occured while scanning", 16)}}
{% endif %}
<form class=delete method="POST">
<input type="hidden" name="action" value="delete_source"/>
<input type="hidden" name="source_id" value="{{source.origin_id}}"/>
<input type="hidden" name="id" value="{{id}}"/>
<button class="icon-button">
{{ utils.show_icon("trash-2", "delete source", 16) }}
</button>
</form>
</div>
{% endfor %}
<span class=spacer></span>
{% endfor %}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment