Skip to content

Commit b2a44fa

Browse files
authored
Merge pull request #4 from chicagopcdc/pcdc_dev
Merge from Pcdc dev for version 0.1.1
2 parents 94af737 + dd7bf08 commit b2a44fa

File tree

13 files changed

+241
-91
lines changed

13 files changed

+241
-91
lines changed

.pre-commit-config.yaml

+1-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,9 @@ repos:
77
- repo: https://github.com/pre-commit/pre-commit-hooks
88
rev: v2.5.0
99
hooks:
10-
- id: end-of-file-fixer
1110
- id: no-commit-to-branch
1211
args: [--branch, develop, --branch, master, --pattern, release/.*]
1312
- repo: https://github.com/psf/black
14-
rev: 19.10b0
13+
rev: 20.8b1
1514
hooks:
1615
- id: black

.secrets.baseline

+10-26
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
{
2-
"custom_plugin_paths": [],
32
"exclude": {
43
"files": null,
54
"lines": null
65
},
7-
"generated_at": "2020-07-24T04:35:13Z",
6+
"generated_at": "2020-09-22T21:21:33Z",
87
"plugins_used": [
98
{
109
"name": "AWSKeyDetector"
@@ -62,14 +61,12 @@
6261
"bin/setup_psqlgraph.py": [
6362
{
6463
"hashed_secret": "f0e2d8610edefa0c02b673dcac7964b02ce3e890",
65-
"is_secret": false,
6664
"is_verified": false,
6765
"line_number": 52,
6866
"type": "Basic Auth Credentials"
6967
},
7068
{
7169
"hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9",
72-
"is_secret": false,
7370
"is_verified": false,
7471
"line_number": 105,
7572
"type": "Basic Auth Credentials"
@@ -78,7 +75,6 @@
7875
"bin/setup_transactionlogs.py": [
7976
{
8077
"hashed_secret": "f0e2d8610edefa0c02b673dcac7964b02ce3e890",
81-
"is_secret": false,
8278
"is_verified": false,
8379
"line_number": 13,
8480
"type": "Basic Auth Credentials"
@@ -87,7 +83,6 @@
8783
"sheepdog/dev_settings.py": [
8884
{
8985
"hashed_secret": "739155dcfcb1fc7f98aa38d18ea3d2213ae62b54",
90-
"is_secret": false,
9186
"is_verified": false,
9287
"line_number": 69,
9388
"type": "Base64 High Entropy String"
@@ -96,7 +91,6 @@
9691
"sheepdog/test_settings.py": [
9792
{
9893
"hashed_secret": "27d0f93fe951bda5dc3973ca2a5b420c7dfc387f",
99-
"is_secret": false,
10094
"is_verified": false,
10195
"line_number": 14,
10296
"type": "Secret Keyword"
@@ -105,21 +99,18 @@
10599
"tests/integration/datadict/submission/data/biospec1.json": [
106100
{
107101
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
108-
"is_secret": false,
109102
"is_verified": false,
110103
"line_number": 63,
111104
"type": "Hex High Entropy String"
112105
},
113106
{
114107
"hashed_secret": "e65fd5600e42643c6d77f3c5c3a60114e0d3165d",
115-
"is_secret": false,
116108
"is_verified": false,
117109
"line_number": 72,
118110
"type": "Hex High Entropy String"
119111
},
120112
{
121113
"hashed_secret": "922282cd0fad4bf8ca1c93de95863177d57a55c1",
122-
"is_secret": false,
123114
"is_verified": false,
124115
"line_number": 81,
125116
"type": "Hex High Entropy String"
@@ -128,14 +119,12 @@
128119
"tests/integration/datadict/submission/data/biospec2.json": [
129120
{
130121
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
131-
"is_secret": false,
132122
"is_verified": false,
133123
"line_number": 18,
134124
"type": "Hex High Entropy String"
135125
},
136126
{
137127
"hashed_secret": "e65fd5600e42643c6d77f3c5c3a60114e0d3165d",
138-
"is_secret": false,
139128
"is_verified": false,
140129
"line_number": 27,
141130
"type": "Hex High Entropy String"
@@ -144,7 +133,6 @@
144133
"tests/integration/datadict/submission/data/databundle1.json": [
145134
{
146135
"hashed_secret": "b87375c47433817ed4310ab448cda0f74670ee71",
147-
"is_secret": false,
148136
"is_verified": false,
149137
"line_number": 44,
150138
"type": "Hex High Entropy String"
@@ -153,7 +141,6 @@
153141
"tests/integration/datadict/submission/data/experimental_metadata.json": [
154142
{
155143
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
156-
"is_secret": false,
157144
"is_verified": false,
158145
"line_number": 5,
159146
"type": "Hex High Entropy String"
@@ -162,7 +149,6 @@
162149
"tests/integration/datadict/submission/data/file.json": [
163150
{
164151
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
165-
"is_secret": false,
166152
"is_verified": false,
167153
"line_number": 9,
168154
"type": "Hex High Entropy String"
@@ -171,30 +157,34 @@
171157
"tests/integration/datadict/submission/data/submitted_unaligned_reads.json": [
172158
{
173159
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
174-
"is_secret": false,
175160
"is_verified": false,
176161
"line_number": 10,
177162
"type": "Hex High Entropy String"
178163
}
179164
],
165+
"tests/integration/datadict/submission/test_endpoints.py": [
166+
{
167+
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
168+
"is_verified": false,
169+
"line_number": 1373,
170+
"type": "Hex High Entropy String"
171+
}
172+
],
180173
"tests/integration/datadictwithobjid/submission/data/biospec1.json": [
181174
{
182175
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
183-
"is_secret": false,
184176
"is_verified": false,
185177
"line_number": 63,
186178
"type": "Hex High Entropy String"
187179
},
188180
{
189181
"hashed_secret": "e65fd5600e42643c6d77f3c5c3a60114e0d3165d",
190-
"is_secret": false,
191182
"is_verified": false,
192183
"line_number": 72,
193184
"type": "Hex High Entropy String"
194185
},
195186
{
196187
"hashed_secret": "922282cd0fad4bf8ca1c93de95863177d57a55c1",
197-
"is_secret": false,
198188
"is_verified": false,
199189
"line_number": 81,
200190
"type": "Hex High Entropy String"
@@ -203,14 +193,12 @@
203193
"tests/integration/datadictwithobjid/submission/data/biospec2.json": [
204194
{
205195
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
206-
"is_secret": false,
207196
"is_verified": false,
208197
"line_number": 18,
209198
"type": "Hex High Entropy String"
210199
},
211200
{
212201
"hashed_secret": "e65fd5600e42643c6d77f3c5c3a60114e0d3165d",
213-
"is_secret": false,
214202
"is_verified": false,
215203
"line_number": 27,
216204
"type": "Hex High Entropy String"
@@ -219,7 +207,6 @@
219207
"tests/integration/datadictwithobjid/submission/data/databundle1.json": [
220208
{
221209
"hashed_secret": "b87375c47433817ed4310ab448cda0f74670ee71",
222-
"is_secret": false,
223210
"is_verified": false,
224211
"line_number": 44,
225212
"type": "Hex High Entropy String"
@@ -228,7 +215,6 @@
228215
"tests/integration/datadictwithobjid/submission/data/experimental_metadata.json": [
229216
{
230217
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
231-
"is_secret": false,
232218
"is_verified": false,
233219
"line_number": 5,
234220
"type": "Hex High Entropy String"
@@ -237,7 +223,6 @@
237223
"tests/integration/datadictwithobjid/submission/data/file.json": [
238224
{
239225
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
240-
"is_secret": false,
241226
"is_verified": false,
242227
"line_number": 9,
243228
"type": "Hex High Entropy String"
@@ -246,14 +231,13 @@
246231
"tests/integration/datadictwithobjid/submission/data/submitted_unaligned_reads.json": [
247232
{
248233
"hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
249-
"is_secret": false,
250234
"is_verified": false,
251235
"line_number": 10,
252236
"type": "Hex High Entropy String"
253237
}
254238
]
255239
},
256-
"version": "0.14.1",
240+
"version": "0.13.1",
257241
"word_list": {
258242
"file": null,
259243
"hash": null

pull_request_template.md

-21
This file was deleted.

requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ authlib==0.14.2
22
authutils==4.0.0
33
boto==2.36.0
44
cryptography==2.8
5-
dictionaryutils==3.1.0
5+
dictionaryutils==3.4.0
66
envelopes==0.4
77
Flask==1.1.1
88
flask-cors==3.0.3

sheepdog/blueprint/routes/views/__init__.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ def root_create():
8080
8181
Args:
8282
body (schema_program): input body
83-
83+
8484
Responses:
8585
200: Registered successfully.
8686
403: Unauthorized request.
@@ -149,13 +149,13 @@ def root_create():
149149
def get_dictionary():
150150
"""
151151
Return links to the JSON schema definitions.
152-
152+
153153
Summary:
154154
Get the dictionary schema
155155
156156
Tags:
157157
dictionary
158-
158+
159159
Responses:
160160
200 (schema_links): Success
161161
403: Unauthorized request.
@@ -193,10 +193,10 @@ def get_templates():
193193
194194
Summary:
195195
Get templates for all entity types
196-
196+
197197
Tags:
198198
dictionary
199-
199+
200200
Query Args:
201201
format (str): output format, ``csv`` or ``tsv``, default is tsv
202202
categories (str): list of entities' categories to include in the template

sheepdog/blueprint/routes/views/program/__init__.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -225,13 +225,13 @@ def delete_program(program):
225225
226226
Summary:
227227
Delete a program
228-
228+
229229
Tags:
230230
program
231231
232232
Args:
233233
program (str): |program_id|
234-
234+
235235
Responses:
236236
204: Success.
237237
400: User error.

sheepdog/transactions/submission/transaction.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,7 @@ def message(self):
8080

8181
@property
8282
def submitted_entity_count(self):
83-
"""Returns the number of entities if submission is successful else 0.
84-
85-
"""
83+
"""Returns the number of entities if submission is successful else 0."""
8684

8785
if not self.success:
8886
return 0
@@ -162,8 +160,10 @@ def send_submission_notification_email(self):
162160
from_addr = self.app_config.get("EMAIL_FROM_ADDRESS")
163161
to_addr = self.app_config.get("EMAIL_SUPPORT_ADDRESS")
164162
preformatted = self.app_config.get("EMAIL_NOTIFICATION_SUBMISSION")
165-
subject = "[SUBMISSION] Project {project_id} has been submitted by {user}".format(
166-
project_id=self.project_id, user=self.user.username
163+
subject = (
164+
"[SUBMISSION] Project {project_id} has been submitted by {user}".format(
165+
project_id=self.project_id, user=self.user.username
166+
)
167167
)
168168

169169
number_of_cases = 0

sheepdog/transactions/upload/entity.py

+6
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,12 @@ def _merge_doc_links(self, node):
407407
doc_sk, doc_ids = set(), set()
408408
doc_links = self.doc.get(name, [])
409409

410+
if doc_links is None:
411+
# If links are explicitly set to None,
412+
# remove all the links from the database
413+
setattr(node, name, [])
414+
continue
415+
410416
# Munge to a list
411417
if isinstance(doc_links, dict):
412418
doc_links = [doc_links]

sheepdog/transactions/upload/sub_entities.py

+9-5
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ class FileUploadEntity(UploadEntity):
5858
index service
5959
2) If an id is provided in the submission we can look up that uuid in
6060
index service
61-
2) The hash/file_size combo provided should be unique in the index
61+
3) The hash/file_size combo provided should be unique in the index
6262
service for each file
6363
6464
Handling Relationship Between Graph Node and Indexed File:
@@ -112,7 +112,7 @@ def __init__(self, *args, **kwargs):
112112
self.file_by_hash = None
113113
self.object_id = None
114114
self.urls = []
115-
self.should_update_acl_uploader = False
115+
self.should_update_acl_and_authz = False
116116

117117
def parse(self, doc):
118118
"""
@@ -147,8 +147,12 @@ def get_node_create(self, skip_node_lookup=False):
147147

148148
# file already indexed and object_id provided: data upload flow
149149
if self.use_object_id(self.entity_type) and self.object_id and self.file_exists:
150-
if self._is_valid_hash_size_for_file():
151-
self.should_update_acl_uploader = True
150+
if (
151+
self._is_valid_hash_size_for_file()
152+
and not getattr(self.file_by_uuid, "acl", None)
153+
and not getattr(self.file_by_uuid, "authz", None)
154+
):
155+
self.should_update_acl_and_authz = True
152156
else:
153157
self._set_node_and_file_ids()
154158

@@ -226,7 +230,7 @@ def flush_to_session(self):
226230
try:
227231
if role == "create":
228232
# data upload flow: update the blank record in indexd
229-
if self.should_update_acl_uploader:
233+
if self.should_update_acl_and_authz:
230234
self._update_acl_uploader_for_file()
231235

232236
# Temporary fix to update authz field in index record

sheepdog/utils/transforms/__init__.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,10 @@ def value_to_list_value(self, cls, link_name, prop, value):
193193
r_values = []
194194
for v in l_values:
195195
converted_value = self.convert_link_value(cls, link_name, prop, v)
196-
r_values.append({prop: converted_value})
196+
# only add the prop if there is a link - for example,
197+
# TSV submissions may include empty link columns
198+
if converted_value:
199+
r_values.append({prop: converted_value})
197200
return r_values
198201

199202
@staticmethod

0 commit comments

Comments
 (0)