alihan/trashcan-scientific-paper-index
1
0
Fork 0
mirror of https://gitlab.com/mildlyparallel/trashcan.git synced 2023-04-08 19:00:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

add new fields to backend

Browse Source
This commit is contained in:
Ruslan Kuchumov
2021-06-19 18:28:49 +03:00
parent 61669ce96c
commit 4c78cb6e6d
6 changed files with 209 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
include/orm/Papers.hh
View File

@@ -21,7 +21,10 @@ public:
Year,
Doi,
Isbn,
Arxiv,
Bibtex,
Url,
Notes,
Filename,
Filesize,
Filemime,
@@ -41,7 +44,10 @@ public:
[Year] = "year",
[Doi] = "doi",
[Isbn] = "isbn",
[Arxiv] = "arxiv",
[Bibtex] = "bibtex",
[Url] = "url",
[Notes] = "notes",
[Filename] = "filename",
[Filesize] = "filesize",
[Filemime] = "filemime",
@@ -82,7 +88,10 @@ public:
"filemime TEXT, "
"doi TEXT, "
"isbn TEXT, "
"arxiv TEXT, "
"bibtex TEXT, "
"url TEXT, "
"notes TEXT, "
"created_at INTEGER DEFAULT (strftime('%s', 'now')), "
"updated_at INTEGER DEFAULT (strftime('%s', 'now')), "
"indexed_at INTEGER "
@@ -103,9 +112,10 @@ public:
std::string_view first_author;
std::string_view authors;
uint64_t year = 0;
std::string_view doi;
std::string_view isbn;
std::string_view doi, arxiv, isbn;
std::string_view bibtex;
std::string_view url;
std::string_view notes;
std::string_view filename;
uint64_t filesize = 0;
std::string_view filemime;
@@ -125,11 +135,14 @@ public:
"year, "
"doi, "
"isbn, "
"arxiv, "
"bibtex, "
"url, "
"notes, "
"filename, "
"filesize, "
"filemime "
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";
Insert(Database &db)
: PaperFields(db, sql)
@@ -165,10 +178,13 @@ public:
bind_to(3, year);
bind_to(4, doi);
bind_to(5, isbn);
bind_to(6, bibtex);
bind_to(7, fn);
bind_to(8, filesize);
bind_to(9, filemime);
bind_to(6, arxiv);
bind_to(7, bibtex);
bind_to(8, url);
bind_to(9, notes);
bind_to(10, fn);
bind_to(11, filesize);
bind_to(12, filemime);
step();
@@ -188,7 +204,10 @@ public:
"year = ?, "
"doi = ?, "
"isbn = ?, "
"arxiv = ?, "
"bibtex = ?, "
"url = ?, "
"notes = ?, "
"updated_at = (strftime('%s', 'now')) "
"WHERE paper_id = ?;";
@@ -208,8 +227,11 @@ public:
bind_to(3, year);
bind_to(4, doi);
bind_to(5, isbn);
bind_to(6, bibtex);
bind_to(7, paper_id);
bind_to(6, arxiv);
bind_to(7, bibtex);
bind_to(8, url);
bind_to(9, notes);
bind_to(10, paper_id);
step();
}
@@ -296,7 +318,10 @@ public:
"indexed_at, "
"doi, "
"isbn, "
"bibtex "
"arxiv, "
"bibtex, "
"url, "
"notes "
"FROM Papers "
"WHERE Papers.paper_id = ?; ";
@@ -313,6 +338,9 @@ public:
std::string doi;
std::string isbn;
std::string bibtex;
std::string arxiv;
std::string url;
std::string notes;
inline void bind(uint64_t paper_id)
{
@@ -338,6 +366,9 @@ public:
load_at(11, doi);
load_at(12, isbn);
load_at(13, bibtex);
load_at(14, arxiv);
load_at(15, url);
load_at(16, notes);
return true;
}
@@ -358,7 +389,10 @@ public:
make_pair(IndexedAt, indexed_at),
make_pair(Doi, doi),
make_pair(Isbn, isbn),
make_pair(Bibtex, bibtex)
make_pair(Bibtex, bibtex),
make_pair(Arxiv, arxiv),
make_pair(Url, url),
make_pair(Notes, notes)
);
}
};
@@ -584,6 +618,5 @@ public:
}
};
};

18
include/orm/PapersIndex.hh
View File

@@ -22,6 +22,8 @@ public:
"year, "
"doi, "
"isbn, "
"arxiv, "
"notes, "
"content "
"); ";
@@ -41,13 +43,15 @@ public:
std::string_view year;
std::string_view doi;
std::string_view isbn;
std::string_view arxiv;
std::string_view bibtex;
std::string_view notes;
};
struct Insert : public Fields {
static const constexpr char *sql =
"INSERT INTO PapersIndex (paper_id, title, authors, year, doi, isbn) "
"VALUES (?, ?, ?, ?, ?, ?);";
"INSERT INTO PapersIndex (paper_id, title, authors, year, doi, isbn, arxiv, notes) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?);";
Insert(Database &db)
: Fields(db, sql)
@@ -64,6 +68,8 @@ public:
bind_to(3, year);
bind_to(4, doi);
bind_to(5, isbn);
bind_to(6, arxiv);
bind_to(7, notes);
step();
}
@@ -76,7 +82,9 @@ public:
"authors = ?, "
"year = ?, "
"doi = ?, "
"isbn = ? "
"isbn = ?, "
"arxiv = ?, "
"notes = ? "
"WHERE paper_id = ?;";
Update(Database &db)
@@ -93,7 +101,9 @@ public:
bind_to(2, year);
bind_to(3, doi);
bind_to(4, isbn);
bind_to(5, paper_id);
bind_to(5, arxiv);
bind_to(6, notes);
bind_to(7, paper_id);
step();
}

18
include/orm/PapersSelect.hh
View File

@@ -45,7 +45,10 @@ public:
"indexed_at, "
"doi, "
"isbn, "
"bibtex "
"bibtex, "
"arxiv, "
"url, "
"notes "
"FROM Papers "
"WHERE Papers.paper_id = ?; ";
@@ -59,9 +62,10 @@ public:
uint64_t created_at;
uint64_t updated_at;
uint64_t indexed_at;
std::string doi;
std::string isbn;
std::string doi, isbn, arxiv;
std::string bibtex;
std::string url;
std::string notes;
inline void bind(uint64_t paper_id)
{
@@ -87,6 +91,9 @@ public:
load_at(11, doi);
load_at(12, isbn);
load_at(13, bibtex);
load_at(14, arxiv);
load_at(15, url);
load_at(16, notes);
return true;
}
@@ -107,7 +114,10 @@ public:
make_pair(IndexedAt, indexed_at),
make_pair(Doi, doi),
make_pair(Isbn, isbn),
make_pair(Bibtex, bibtex)
make_pair(Bibtex, bibtex),
make_pair(Arxiv, arxiv),
make_pair(Url, url),
make_pair(Notes, notes)
);
}
};

38
migrations/v1.1-to-v1.2.sql Normal file
View File

@@ -0,0 +1,38 @@
BEGIN TRANSACTION;
ALTER TABLE Papers ADD COLUMN url TEXT;
ALTER TABLE Papers ADD COLUMN arxiv TEXT;
ALTER TABLE Papers ADD COLUMN notes TEXT;
DROP TABLE IF EXISTS NewPapersIndex;
CREATE VIRTUAL TABLE NewPapersIndex
USING fts4(
paper_id REFERENCES Papers(paper_id),
title,
authors,
year,
doi,
isbn,
arxiv,
notes,
content
);
INSERT INTO NewPapersIndex(
paper_id,
title,
authors,
year,
doi,
isbn,
content
)
SELECT paper_id, title, authors, year, doi, isbn, content
FROM PapersIndex;
DROP TABLE PapersIndex;
ALTER TABLE NewPapersIndex RENAME TO PapersIndex;
COMMIT;

35
src/TrashServer.cc
View File

@@ -244,19 +244,31 @@ void TrashServer::create_paper(const Request &req)
} else if (name == Papers::names[Papers::Title]) {
paper.title = body;
index.title = body;
} else if (name == Papers::names[Papers::FirstAuthor]) {
paper.first_author = body;
} else if (name == Papers::names[Papers::Authors]) {
paper.authors = body;
index.authors = body;
size_t comma_pos = body.find(',');
if (comma_pos == body.npos)
paper.first_author = body;
else
paper.first_author = body.substr(0, comma_pos);
} else if (name == Papers::names[Papers::Doi]) {
paper.doi = body;
index.doi = body;
} else if (name == Papers::names[Papers::Isbn]) {
paper.isbn = body;
index.isbn = body;
} else if (name == Papers::names[Papers::Arxiv]) {
paper.arxiv = body;
index.arxiv = body;
} else if (name == Papers::names[Papers::Bibtex]) {
paper.bibtex = body;
} else if (name == Papers::names[Papers::Url]) {
paper.url = body;
} else if (name == Papers::names[Papers::Notes]) {
paper.notes = body;
index.notes = body;
} else if (name == Papers::names[Papers::Year]) {
try {
paper.year = std::stoull(std::string(body));
@@ -318,11 +330,16 @@ void TrashServer::update_paper(const Request &req)
if (name == Papers::names[Papers::Title]) {
paper.title = body;
index.title = body;
} else if (name == Papers::names[Papers::FirstAuthor]) {
paper.first_author = body;
} else if (name == Papers::names[Papers::Authors]) {
paper.authors = body;
index.authors = body;
size_t comma_pos = body.find(',');
if (comma_pos == body.npos)
paper.first_author = body;
else
paper.first_author = body.substr(0, comma_pos);
} else if (name == Papers::names[Papers::Doi]) {
paper.doi = body;
index.doi = body;
@@ -331,6 +348,16 @@ void TrashServer::update_paper(const Request &req)
index.isbn = body;
} else if (name == Papers::names[Papers::Bibtex]) {
paper.bibtex = body;
} else if (name == Papers::names[Papers::Arxiv]) {
paper.arxiv = body;
index.arxiv = body;
} else if (name == Papers::names[Papers::Bibtex]) {
paper.bibtex = body;
} else if (name == Papers::names[Papers::Url]) {
paper.url = body;
} else if (name == Papers::names[Papers::Notes]) {
paper.notes = body;
index.notes = body;
} else if (name == Papers::names[Papers::Year]) {
try {
paper.year = std::stoull(std::string(body));

90
tests/test-papers.py
View File

@@ -50,22 +50,49 @@ class TestTags(RestTest):
return tag
def test__post_get__all_fields__returns_err(self):
def __generate_fields(self):
paper = {
'title': self._genstr(),
'authors': ','.join([self._genstr(), self._genstr(), self._genstr()]),
'doi': self._genstr(),
'isbn': self._genstr(),
'arxiv': self._genstr(),
'url': self._genstr(),
'notes': self._genstr(),
'bibtex': self._genstr(),
'year': random.randint(1970, 9999)
}
return paper
def __check_paper_fields(self, body, paper, tags):
for k, v in paper.items():
if k == 'tags':
continue
self.assertEqual(body['papers'][0][k], paper[k]);
self.assertIn('created_at', body['papers'][0])
self.assertIn('updated_at', body['papers'][0])
self.assertIn('indexed_at', body['papers'][0])
authors = paper['authors'].split(',')
self.assertEqual(body['papers'][0]['first_author'], authors[0]);
if tags:
self.assertEqual(len(body['tags']), len(tags))
for tag in body['tags']:
t = { k: tag[k] for k in ['tag_id', 'name', 'color'] }
self.assertIn(t, tags)
def test__create__all_fields(self):
tags = [
self.__post_tag(),
self.__post_tag(),
]
paper = {
'title': self._genstr(),
'authors': self._genstr(),
'first_author': self._genstr(),
'doi': self._genstr(),
'isbn': self._genstr(),
'bibtex': self._genstr(),
'year': 2020,
'tags': [ t['tag_id'] for t in tags ]
}
paper = self.__generate_fields()
paper['tags'] = [ t['tag_id'] for t in tags ]
(code, body) = self._post("/papers", paper, '/etc/hosts')
@@ -77,22 +104,39 @@ class TestTags(RestTest):
(code, body) = self._get(f'papers/{paper_id}')
self.assertEqual(code, 200);
for k, v in paper.items():
if k == 'tags':
continue
self.assertEqual(body['papers'][0][k], paper[k]);
self.__check_paper_fields(body, paper, tags)
self.assertTrue(body['papers'][0]['filesize'] > 0);
self.assertEqual(body['papers'][0]['filemime'], 'text/plain');
self.assertIn('created_at', body['papers'][0])
self.assertIn('updated_at', body['papers'][0])
self.assertIn('indexed_at', body['papers'][0])
self.assertEqual(len(body['tags']), len(tags))
for tag in body['tags']:
t = { k: tag[k] for k in ['tag_id', 'name', 'color'] }
self.assertIn(t, tags)
def test__update__all_fields(self):
tags = [
self.__post_tag(),
self.__post_tag(),
]
paper = self.__generate_fields()
paper['tags'] = [ t['tag_id'] for t in tags ]
(code, body) = self._post("/papers", paper, '/etc/hosts')
self.assertEqual(code, 200);
self.assertTrue(body['paper_id'] > 0);
paper_id = body['paper_id']
paper = self.__generate_fields()
(code, body) = self._post(f"/papers/{paper_id}", paper, '/etc/passwd')
(code, body) = self._get(f'papers/{paper_id}')
self.assertEqual(code, 200);
self.__check_paper_fields(body, paper, tags)
self.assertTrue(body['papers'][0]['filesize'] > 0);
self.assertEqual(body['papers'][0]['filemime'], 'text/plain');
if __name__ == '__main__':
unittest.main()