alihan/zotprime
1
0
Fork 0
mirror of https://github.com/ZotPrime/zotprime.git synced 2021-05-12 18:32:25 +03:00
Code Issues Projects Releases Wiki Activity

Goodbye Mongo, hello denormalized MySQL

Browse Source 
- Store itemDataValues and creatorData directly in itemData and creators
- Hashes to be removed after migration
This commit is contained in:
Dan Stillman
2011-09-04 20:34:55 +00:00
parent 076ecb160e
commit 555dd0965c
14 changed files with 255 additions and 834 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
controllers/ApiController.php
View File

@@ -56,7 +56,10 @@ class ApiController extends Controller {
private $httpAuth;
private $profile = false;
private $profileShard = 0;
private $profileShard = 1;
private $startTime = false;
private $timeLogged = false;
private $timeLogThreshold = 5;
public function __construct($action, $settings, $extra) {
@@ -67,6 +70,7 @@ class ApiController extends Controller {
set_exception_handler(array($this, 'handleException'));
require_once('../model/Error.inc.php');
$this->startTime = microtime(true);
$this->method = $_SERVER['REQUEST_METHOD'];
if (!in_array($this->method, array('HEAD', 'GET', 'PUT', 'POST', 'DELETE'))) {
@@ -322,16 +326,6 @@ class ApiController extends Controller {
$this->e400("$this->method data not provided");
}
// TEMP
//$mongo = !empty($_GET['mongo']);
//$mongo = Z_CONFIG::$TESTING_SITE;
$mongo = false;
// For now, force Mongo mode for itemKey requests, which should come only
// from post-write redirections
if (!empty($this->queryParams['itemKey'])) {
$mongo = true;
}
$itemIDs = array();
$items = array();
$totalResults = null;
@@ -640,12 +634,7 @@ class ApiController extends Controller {
$this->allowMethods(array('GET'));
$title = "Top-Level Items";
if ($mongo) {
$results = Zotero_Items::searchMongo($this->objectLibraryID, true, $this->queryParams);
}
else {
$results = Zotero_Items::searchMySQL($this->objectLibraryID, true, $this->queryParams);
}
$results = Zotero_Items::search($this->objectLibraryID, true, $this->queryParams);
}
else if ($this->subset == 'trash') {
$this->allowMethods(array('GET'));
@@ -706,8 +695,6 @@ class ApiController extends Controller {
$this->responseCode = 201;
$this->queryParams = Zotero_API::parseQueryParams($queryString);
// TEMP
$mongo = true;
}
// Display items
@@ -746,26 +733,15 @@ class ApiController extends Controller {
$this->responseCode = 201;
$this->queryParams = Zotero_API::parseQueryParams($queryString);
// TEMP
$mongo = true;
}
$title = "Items";
// TEMP
if ($mongo) {
$results = Zotero_Items::searchMongo($this->objectLibraryID, false, $this->queryParams);
}
else {
$results = Zotero_Items::searchMySQL($this->objectLibraryID, false, $this->queryParams);
}
$results = Zotero_Items::search($this->objectLibraryID, false, $this->queryParams);
}
// TEMP
if (!$mongo) {
if (!empty($results)) {
$items = $results['items'];
$totalResults = $results['total'];
}
if (!empty($results)) {
$items = $results['items'];
$totalResults = $results['total'];
}
}
@@ -776,15 +752,8 @@ class ApiController extends Controller {
}
if ($itemIDs) {
// TEMP
if ($mongo) {
$this->queryParams['dbkeys'] = Zotero_Items::idsToKeys($this->objectLibraryID, $itemIDs);
$results = Zotero_Items::searchMongo($this->objectLibraryID, false, $this->queryParams, $includeTrashed);
}
else {
$this->queryParams['dbkeys'] = Zotero_Items::idsToKeys($this->objectLibraryID, $itemIDs);
$results = Zotero_Items::searchMySQL($this->objectLibraryID, false, $this->queryParams, $includeTrashed);
}
$this->queryParams['itemIDs'] = $itemIDs;
$results = Zotero_Items::search($this->objectLibraryID, false, $this->queryParams, $includeTrashed);
$items = $results['items'];
$totalResults = $results['total'];
@@ -2574,11 +2543,33 @@ class ApiController extends Controller {
echo $xmlstr;
$this->logRequestTime();
echo ob_get_clean();
exit;
}
private function currentRequestTime() {
return microtime(true) - $this->startTime;
}
private function logRequestTime($point=false) {
if ($this->timeLogged) {
return;
}
$time = $this->currentRequestTime();
if ($time > $this->timeLogThreshold) {
$this->timeLogged = true;
error_log(
"Slow API request " . ($point ? " at point " . $point : "") . ": "
. $time . " sec for "
. $_SERVER['REQUEST_METHOD'] . " " . $_SERVER['REQUEST_URI']
);
}
}
private function jsonDecode($json) {
$obj = json_decode($json);

8
misc/shard.sql
View File

@@ -60,7 +60,10 @@ CREATE TABLE `collections` (
CREATE TABLE `creators` (
`creatorID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`libraryID` int(10) unsigned NOT NULL,
`creatorDataHash` char(32) CHARACTER SET ascii NOT NULL,
`creatorDataHash` char(32) CHARACTER SET ascii DEFAULT NULL,
`firstName` varchar(255) DEFAULT NULL,
`lastName` varchar(255) DEFAULT NULL,
`fieldMode` tinyint(1) unsigned DEFAULT NULL,
`dateAdded` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`dateModified` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`key` char(8) NOT NULL,
@@ -123,7 +126,8 @@ CREATE TABLE `itemCreators` (
CREATE TABLE `itemData` (
`itemID` int(10) unsigned NOT NULL,
`fieldID` smallint(5) unsigned NOT NULL,
`itemDataValueHash` char(32) CHARACTER SET ascii NOT NULL,
`itemDataValueHash` char(32) CHARACTER SET ascii DEFAULT NULL,
`value` text,
PRIMARY KEY (`itemID`,`fieldID`),
KEY `fieldID` (`fieldID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

16
model/Creator.inc.php
View File

@@ -28,7 +28,6 @@ class Zotero_Creator {
private $id;
private $libraryID;
private $key;
private $creatorDataHash;
private $firstName = '';
private $lastName = '';
private $shortName = '';
@@ -51,7 +50,6 @@ class Zotero_Creator {
private function init() {
$this->creatorDataHash = false;
$this->loaded = false;
$this->changed = array();
@@ -164,17 +162,15 @@ class Zotero_Creator {
Z_Core::debug("Saving creator $this->id");
$key = $this->key ? $this->key : $this->generateKey();
$creatorDataHash = Zotero_Creators::getDataHash($this, true);
$timestamp = Zotero_DB::getTransactionTimestamp();
$dateAdded = $this->dateAdded ? $this->dateAdded : $timestamp;
$dateModified = $this->changed['dateModified'] ? $this->dateModified : $timestamp;
$fields = "creatorDataHash=?, firstName=?, lastName=?, fieldMode=?,
$fields = "firstName=?, lastName=?, fieldMode=?,
libraryID=?, `key`=?, dateAdded=?, dateModified=?, serverDateModified=?";
$params = array(
$creatorDataHash,
$this->firstName,
$this->lastName,
$this->fieldMode,
@@ -219,7 +215,9 @@ class Zotero_Creator {
'key' => $key,
'dateAdded' => $dateAdded,
'dateModified' => $dateModified,
'creatorDataHash' => $creatorDataHash
'firstName' => $this->firstName,
'lastName' => $this->lastName,
'fieldMode' => $this->fieldMode
)
);
}
@@ -237,7 +235,6 @@ class Zotero_Creator {
}
$this->init();
$this->creatorDataHash = $creatorDataHash;
if ($isNew) {
Zotero_Creators::cache($this);
@@ -293,11 +290,6 @@ class Zotero_Creator {
foreach ($row as $key=>$val) {
$this->$key = $val;
}
$data = Zotero_Creators::getData($row['creatorDataHash']);
foreach ($data as $key=>$val) {
$this->$key = $val;
}
}

6
model/Creators.inc.php
View File

@@ -25,6 +25,8 @@
*/
class Zotero_Creators extends Zotero_DataObjects {
public static $creatorSummarySortLength = 50;
protected static $ZDO_object = 'creator';
private static $fields = array(
@@ -169,8 +171,8 @@ class Zotero_Creators extends Zotero_DataObjects {
public static function getPrimaryDataSQL() {
return "SELECT creatorID AS id, libraryID, `key`, dateAdded, dateModified, creatorDataHash
FROM creators WHERE ";
return "SELECT creatorID AS id, libraryID, `key`, dateAdded, dateModified,
firstName, lastName, fieldMode FROM creators WHERE ";
}

11
model/DataObjects.inc.php
View File

@@ -299,7 +299,7 @@ class Zotero_DataObjects {
}
$found = 0;
$expected = 6; // number of values below
$expected = 8; // number of values below
foreach ($row as $key=>$val) {
switch ($key) {
@@ -308,7 +308,9 @@ class Zotero_DataObjects {
case 'key':
case 'dateAdded':
case 'dateModified':
case 'creatorDataHash':
case 'firstName':
case 'lastName':
case 'fieldMode':
$found++;
break;
@@ -519,11 +521,6 @@ class Zotero_DataObjects {
$timestamp = Zotero_DB::getTransactionTimestamp();
$params = array($libraryID, $key, $timestamp, $timestamp);
Zotero_DB::query($sql, $params, $shardID);
// Queue item for deletion from search index
if ($type == 'item') {
Zotero_Index::queueItem($libraryID, $key);
}
}
Zotero_DB::commit();

3
model/Group.inc.php
View File

@@ -706,9 +706,6 @@ class Zotero_Group {
Z_Core::logError($e);
}
// Queue library for deletion from search index
Zotero_Index::queueLibrary($this->libraryID);
Zotero_DB::commit();
$this->erased = true;

219
model/Index.inc.php
View File

@@ -1,219 +0,0 @@
<?
class Zotero_Index {
public static $queueingEnabled = true;
private static $inTransaction = false;
private static $pairsToCommit = array();
public static function addItem($item) {
self::addItems(array($item));
}
public static function addItems($items) {
foreach ($items as $item) {
$doc = $item->toMongoIndexDocument();
$doc['ts'] = new MongoDate();
Z_Core::$Mongo->update("searchItems", $doc["_id"], $doc, array("upsert"=>true, "safe"=>true));
}
}
public static function removeItem($libraryID, $key) {
Z_Core::$Mongo->remove("searchItems", "$libraryID/$key");
}
public static function removeItems($pairs) {
$ids = array();
foreach ($pairs as $pair) {
$ids[] = $pair['libraryID'] . "/" . $pair['key'];
}
Z_Core::$Mongo->remove("searchItems", array("_id" => array('$in'=>$ids)));
}
public static function removeLibrary($libraryID) {
$re = new MongoRegex('/^' . $libraryID . '\//');
Z_Core::$Mongo->remove("searchItems", array("_id" => $re));
}
public static function removeLibraries($libraryIDs) {
foreach ($libraryIDs as $libraryID) {
self::removeLibrary($libraryID);
}
}
/**
* Start holding libraryID/key pairs for batched insert into queue table
*/
public static function begin() {
if (self::$inTransaction) {
throw new Exception("Already in a transaction");
}
self::$inTransaction = true;
}
/**
* Batch insert libraryID/key pairs into queue table
*/
public static function commit() {
if (!self::$inTransaction) {
throw new Exception("Not in a transaction");
}
self::$inTransaction = false;
self::queueItems(self::$pairsToCommit);
}
public static function rollback() {
if (!self::$inTransaction) {
Z_Core::debug('Transaction not open in Zotero_Index::rollback()');
return;
}
self::$pairsToCommit = array();
self::$inTransaction = false;
}
public static function queueItem($libraryID, $key) {
self::queueItems(array(array($libraryID, $key)));
}
public static function queueItems($pairs) {
if (!self::$queueingEnabled) {
return;
}
// Allow batched inserts
if (self::$inTransaction) {
for ($i=0, $len=sizeOf($pairs); $i<$len; $i++) {
self::$pairsToCommit[] = $pairs[$i];
}
return;
}
$sql = "INSERT IGNORE INTO indexQueue (libraryID, `key`) VALUES ";
Zotero_DB::bulkInsert($sql, $pairs, 100);
}
public static function queueLibrary($libraryID) {
if (!self::$queueingEnabled) {
return;
}
$sql = "INSERT IGNORE INTO indexQueue (libraryID, `key`) VALUES (?,?)";
Zotero_DB::query($sql, array($libraryID, ''));
}
public static function getQueuedItems($indexProcessID, $max=100) {
$sql = "UPDATE indexQueue SET indexProcessID=?
WHERE indexProcessID IS NULL
ORDER BY added LIMIT $max";
Zotero_DB::query($sql, $indexProcessID);
$sql = "SELECT libraryID, `key` FROM indexQueue WHERE indexProcessID=?";
$rows = Zotero_DB::query($sql, $indexProcessID);
if (!$rows) {
$rows = array();
}
return $rows;
}
public static function processFromQueue($indexProcessID) {
// Update host field with the host processing the data
$addr = gethostbyname(gethostname());
$sql = "INSERT INTO indexProcesses (indexProcessID, processorHost) VALUES (?, INET_ATON(?))";
Zotero_DB::query($sql, array($indexProcessID, $addr));
$updateItems = array();
$deletePairs = array();
$deleteLibraries = array();
$libraryExists = array();
$lkPairs = self::getQueuedItems($indexProcessID);
foreach ($lkPairs as $pair) {
if (!isset($libraryExists[$pair['libraryID']])) {
$libraryExists[$pair['libraryID']] = Zotero_Libraries::exists($pair['libraryID']);
}
// If key not specified or library doesn't exist, update/delete entire library
if (!$pair['key'] || !$libraryExists[$pair['libraryID']]) {
if ($libraryExists[$pair['libraryID']]) {
throw new Exception("Unimplemented");
continue;
}
// Delete by query
$deleteLibraries[] = $pair['libraryID'];
continue;
}
$item = Zotero_Items::getByLibraryAndKey($pair['libraryID'], $pair['key']);
if (!$item) {
$deletePairs[] = $pair;
continue;
}
$updateItems[] = $item;
}
try {
if ($updateItems) {
self::addItems($updateItems);
}
if ($deletePairs) {
self::removeItems($deletePairs);
}
if ($deleteLibraries) {
self::removeLibraries($deleteLibraries);
}
}
catch (Exception $e) {
Z_Core::logError($e);
self::removeProcess($indexProcessID);
return -2;
}
self::removeQueuedItems($indexProcessID);
self::removeProcess($indexProcessID);
return 1;
}
public static function removeQueuedItems($indexProcessID) {
$sql = "DELETE FROM indexQueue WHERE indexProcessID=?";
Zotero_DB::query($sql, $indexProcessID);
}
public static function countQueuedProcesses() {
$sql = "SELECT COUNT(*) FROM indexQueue";
return Zotero_DB::valueQuery($sql);
}
public static function getOldProcesses($host=null, $seconds=60) {
$sql = "SELECT DISTINCT indexProcessID FROM indexProcesses
WHERE started < NOW() - INTERVAL ? SECOND";
$params = array($seconds);
if ($host) {
$sql .= " AND processorHost=INET_ATON(?)";
$params[] = $host;
}
return Zotero_DB::columnQuery($sql, $params);
}
public static function removeProcess($indexProcessID) {
$sql = "DELETE FROM indexProcesses WHERE indexProcessID=?";
Zotero_DB::query($sql, $indexProcessID);
}
}
?>

218
model/Item.inc.php
View File

@@ -219,6 +219,7 @@ class Zotero_Item {
return $this->$field;
}
if ($this->isNote()) {
switch ($field) {
case 'title':
@@ -1184,15 +1185,15 @@ class Zotero_Item {
//
if ($this->changed['itemData']) {
// Use manual bound parameters to speed things up
$origInsertSQL = "INSERT INTO itemData VALUES ";
$origInsertSQL = "INSERT INTO itemData (itemID, fieldID, value) VALUES ";
$insertSQL = $origInsertSQL;
$insertParams = array();
$insertCounter = 0;
$maxInsertGroups = 40;
$fieldIDs = array_keys($this->changed['itemData']);
$max = Zotero_Items::$maxDataValueLength;
$lastFieldID = $fieldIDs[sizeOf($fieldIDs) - 1];
$fieldIDs = array_keys($this->changed['itemData']);
foreach ($fieldIDs as $fieldID) {
$value = $this->getField($fieldID, true, false, true);
@@ -1202,27 +1203,20 @@ class Zotero_Item {
$value = Zotero_DB::getTransactionTimestamp();
}
try {
$last = $fieldID == $lastFieldID;
$hash = Zotero_Items::getDataValueHash($value, true, $last);
}
catch (Exception $e) {
$msg = $e->getMessage();
if (strpos($msg, "Data too long for column 'value'") !== false) {
$fieldName = Zotero_ItemFields::getLocalizedString(
$this->itemTypeID, $fieldID
);
throw new Exception("=$fieldName field " .
"'" . substr($value, 0, 50) . "...' too long");
}
throw ($e);
// Check length
if (strlen($value) > $max) {
$fieldName = Zotero_ItemFields::getLocalizedString(
$this->itemTypeID, $fieldID
);
throw new Exception("=$fieldName field " .
"'" . substr($value, 0, 50) . "...' too long");
}
if ($insertCounter < $maxInsertGroups) {
$insertSQL .= "(?,?,?,?),";
$insertSQL .= "(?,?,?),";
$insertParams = array_merge(
$insertParams,
array($itemID, $fieldID, $hash, $value)
array($itemID, $fieldID, $value)
);
}
@@ -1253,7 +1247,6 @@ class Zotero_Item {
Z_Core::$MC->set("itemUsedFieldNames_" . $itemID, $names);
}
//
// Creators
//
@@ -1403,6 +1396,11 @@ class Zotero_Item {
Zotero_DB::query($sql, $bindParams, $shardID);
}
// Sort fields
$sortTitle = Zotero_Items::getSortTitle($this->getDisplayTitle(true));
$creatorSummary = $this->isRegularItem() ? substr($this->getCreatorSummary(), 0, Zotero_Creators::$creatorSummarySortLength) : '';
$sql = "INSERT INTO itemSortFields (itemID, sortTitle, creatorSummary) VALUES (?, ?, ?)";
Zotero_DB::query($sql, array($itemID, $sortTitle, $creatorSummary), $shardID);
//
// Source item id
@@ -1608,15 +1606,15 @@ class Zotero_Item {
if ($this->changed['itemData']) {
$del = array();
$origReplaceSQL = "REPLACE INTO itemData VALUES ";
$origReplaceSQL = "REPLACE INTO itemData (itemID, fieldID, value) VALUES ";
$replaceSQL = $origReplaceSQL;
$replaceParams = array();
$replaceCounter = 0;
$maxReplaceGroups = 40;
$fieldIDs = array_keys($this->changed['itemData']);
$max = Zotero_Items::$maxDataValueLength;
$lastFieldID = $fieldIDs[sizeOf($fieldIDs) - 1];
$fieldIDs = array_keys($this->changed['itemData']);
foreach ($fieldIDs as $fieldID) {
$value = $this->getField($fieldID, true, false, true);
@@ -1632,24 +1630,17 @@ class Zotero_Item {
$value = Zotero_DB::getTransactionTimestamp();
}
try {
$last = $fieldID == $lastFieldID;
$hash = Zotero_Items::getDataValueHash($value, true, $last);
}
catch (Exception $e) {
$msg = $e->getMessage();
if (strpos($msg, "Data too long for column 'value'") !== false) {
$fieldName = Zotero_ItemFields::getLocalizedString(
$this->itemTypeID, $fieldID
);
throw new Exception("=$fieldName field " .
"'" . substr($value, 0, 50) . "...' too long");
}
throw ($e);
// Check length
if (strlen($value) > $max) {
$fieldName = Zotero_ItemFields::getLocalizedString(
$this->itemTypeID, $fieldID
);
throw new Exception("=$fieldName field " .
"'" . substr($value, 0, 50) . "...' too long");
}
if ($replaceCounter < $maxReplaceGroups) {
$replaceSQL .= "(?,?,?,?),";
$replaceSQL .= "(?,?,?),";
$replaceParams = array_merge($replaceParams,
array($this->id, $fieldID, $hash, $value)
);
@@ -1861,6 +1852,28 @@ class Zotero_Item {
Zotero_DB::query($sql, $bindParams, $shardID);
}
// Sort fields
if (!empty($this->changed['primaryData']['itemTypeID']) || $this->changed['itemData'] || $this->changed['creators']) {
$sql = "UPDATE itemSortFields SET sortTitle=?";
$params = array();
$sortTitle = Zotero_Items::getSortTitle($this->getDisplayTitle(true));
$params[] = $sortTitle;
if ($this->changed['creators']) {
$creatorSummary = $this->isRegularItem() ? substr($this->getCreatorSummary(), 0, Zotero_Creators::$creatorSummarySortLength) : '';
$sql .= ", creatorSummary=?";
$params[] = $creatorSummary;
}
$sql .= " WHERE itemID=?";
$params[] = $itemID;
Zotero_DB::query($sql, $params, $shardID);
}
//
// Source item id
//
@@ -2039,9 +2052,6 @@ class Zotero_Item {
// TODO: invalidate memcache
Zotero_Items::reload($this->libraryID, $this->id);
// Queue item for addition to search index
Zotero_Index::queueItem($this->libraryID, $this->key);
if ($isNew) {
//Zotero.Notifier.trigger('add', 'item', $this->getID());
return $this->id;
@@ -3263,118 +3273,6 @@ class Zotero_Item {
}
public function toMongoIndexDocument() {
if (!$this->loaded['primaryData']) {
$this->loadPrimaryData(true);
}
$fields = array();
$fields['_id'] = $this->libraryID . "/" . $this->key;
$fields['dateAdded'] = new MongoDate(strtotime($this->dateAdded));
$fields['dateModified'] = new MongoDate(strtotime($this->dateModified));
$fields['serverDateModified'] = strtotime($this->serverDateModified);
if ($parent = $this->getSourceKey()) {
$fields['parent'] = $parent;
}
if ($this->getDeleted()) {
$fields['deleted'] = true;
}
$creatorSummary = $this->getCreatorSummary();
if ($creatorSummary) {
$fields['creatorSummary'] = $creatorSummary;
}
// Store this annoying field until Mongo supports advanced sorting
$fields['creatorIsEmpty'] = !$creatorSummary;
// Title for sorting
$title = $this->getDisplayTitle(true);
$title = $title ? $title : '';
// Strip HTML from note titles
if ($this->isNote()) {
// Notes don't get titles automatically
$fields['title'] = $title;
}
// Strip some characters
$sortTitle = preg_replace("/^[\[\'\"]*(.*)[\]\'\"]*$/", "$1", $title);
if ($sortTitle) {
$fields['sortTitle'] = $sortTitle;
}
$itemData = $this->toJSON(true, false, false, true);
if (empty($itemData['note'])) {
unset($itemData['note']);
}
if (empty($itemData['tags'])) {
unset($itemData['tags']);
}
$doc = array_merge($fields, $itemData);
/*
Not doing full-text search for now, since, among other things, splitting
on whitespace and doing left-bound searches wouldn't work for Asian languages,
and in general left-bound searches would require better filtering
// Generate keywords array
$keywords = array();
foreach ($doc as $key=>$val) {
switch ($key) {
case '_id':
case 'dateAdded':
case 'dateModified':
case 'serverDateModified':
case 'deleted':
case 'creatorSummary':
case 'creatorIsEmpty':
case 'sortTitle':
case 'itemType':
case 'parent':
case 'linkMode':
case 'mimeType':
case 'charset':
case 'ts':
continue 2;
}
if ($key == "creators") {
// Turn creator into string that can be separated
$creators = array();
foreach ($val as $creator) {
$creators[] = !empty($creator['name']) ? $creator['name'] : $creator['firstName'] . ' ' . $creator['lastName'];
}
$val = implode(" ", $creators);
}
else if ($key == "note") {
$val = strip_tags(Zotero_Notes::sanitize($val));
// Unencode plaintext string
$val = html_entity_decode($val);
}
else if ($key == "tags") {
$tags = array();
foreach ($val as $tag) {
$tags[] = $tag['tag'];
}
$val = implode(" ", $tags);
}
$words = preg_split('/\s+/', trim($val));
foreach ($words as $word) {
// Skip one-letter words
if (strlen($word) == 1 && preg_match('/^[\x{20}-\x{FF}]/u', $word)) {
continue;
}
$keywords[] = strtolower($word);
}
}
$doc['keywords'] = array_values(array_unique($keywords));*/
return $doc;
}
public function toSolrDocument() {
$doc = new SolrInputDocument();
@@ -3591,25 +3489,15 @@ class Zotero_Item {
trigger_error("Invalid itemID '$this->id'", E_USER_ERROR);
}
$sql = "SELECT fieldID, itemDataValueHash AS hash FROM itemData WHERE itemID=?";
$sql = "SELECT fieldID, value FROM itemData WHERE itemID=?";
$stmt = Zotero_DB::getStatement($sql, true, Zotero_Shards::getByLibraryID($this->libraryID));
$fields = Zotero_DB::queryFromStatement($stmt, $this->id);
$itemTypeFields = Zotero_ItemFields::getItemTypeFields($this->itemTypeID);
if ($fields) {
$hashes = array();
foreach($fields as $field) {
$hashes[] = $field['hash'];
}
$values = Zotero_Items::getDataValues($hashes);
foreach ($fields as $field) {
if (!isset($values[$field['hash']])) {
throw new Exception("Item data value for hash '{$field['hash']}' not found");
}
$this->setField($field['fieldID'], $values[$field['hash']], true, true);
$this->setField($field['fieldID'], $field['value'], true, true);
}
}

434
model/Items.inc.php
View File

@@ -105,22 +105,102 @@ class Zotero_Items extends Zotero_DataObjects {
}
public static function searchMySQL($libraryID, $onlyTopLevel=false, $params=array(), $includeTrashed=false) {
public static function search($libraryID, $onlyTopLevel=false, $params=array(), $includeTrashed=false) {
$results = array('items' => array(), 'total' => 0);
$shardID = Zotero_Shards::getByLibraryID($libraryID);
$sql = "SELECT SQL_CALC_FOUND_ROWS I.itemID FROM items I ";
$itemIDs = array();
$keys = array();
// Pass a list of itemIDs, for when the initial search is done via SQL
if (!empty($params['itemIDs'])) {
$itemIDs = $params['itemIDs'];
}
if (!empty($params['itemKey'])) {
$keys = explode(',', $params['itemKey']);
}
$sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT I.itemID FROM items I ";
$sqlParams = array($libraryID);
if ($onlyTopLevel || (!empty($params['order']) && $params['order'] == 'title')) {
$sql .= "LEFT JOIN itemNotes INo USING (itemID) ";
if (!empty($params['q'])) {
$titleFieldIDs = array_merge(
array(Zotero_ItemFields::getID('title')),
Zotero_ItemFields::getTypeFieldsFromBase('title')
);
$sql .= "LEFT JOIN itemData ID ON (ID.itemID=I.itemID AND fieldID IN ("
. implode(',', $titleFieldIDs) . ")) ";
$sql .= "LEFT JOIN itemCreators IC ON (IC.itemID=I.itemID)
LEFT JOIN creators C ON (C.creatorID=IC.creatorID) ";
}
if ($onlyTopLevel || !empty($params['q'])) {
$sql .= "LEFT JOIN itemNotes INo ON (INo.itemID=I.itemID) ";
}
if ($onlyTopLevel) {
$sql .= "LEFT JOIN itemAttachments IA ON (IA.itemID=I.itemID) ";
}
if (!$includeTrashed) {
$sql .= " LEFT JOIN deletedItems DI ON (DI.itemID=I.itemID) ";
$sql .= "LEFT JOIN deletedItems DI ON (DI.itemID=I.itemID) ";
}
if (!empty($params['order'])) {
switch ($params['order']) {
case 'title':
case 'creator':
$sql .= "LEFT JOIN itemSortFields ISF ON (ISF.itemID=I.itemID) ";
break;
case 'addedBy':
$isGroup = Zotero_Libraries::getType($libraryID) == 'group';
if ($isGroup) {
// Create temporary table to store usernames
//
// We use IF NOT EXISTS just to make sure there are
// no problems with restoration from the binary log
$sql2 = "CREATE TEMPORARY TABLE IF NOT EXISTS tmpCreatedByUsers
(userID INT UNSIGNED NOT NULL,
username VARCHAR(255) NOT NULL,
PRIMARY KEY (userID),
INDEX (username))";
Zotero_DB::query($sql2, false, $shardID);
$deleteTempTable = true;
$sql2 = "SELECT DISTINCT createdByUserID FROM items
JOIN groupItems USING (itemID) WHERE ";
if ($itemIDs) {
$sql2 .= "itemID IN ("
. implode(', ', array_fill(0, sizeOf($itemIDs), '?'))
. ") ";
$createdByUserIDs = Zotero_DB::columnQuery($sql2, $itemIDs, $shardID);
}
else {
$sql2 .= "libraryID=?";
$createdByUserIDs = Zotero_DB::columnQuery($sql2, $libraryID, $shardID);
}
// Populate temp table with usernames
if ($createdByUserIDs) {
$toAdd = array();
foreach ($createdByUserIDs as $createdByUserID) {
$toAdd[] = array(
$createdByUserID,
Zotero_Users::getUsername($createdByUserID)
);
}
$sql2 = "INSERT IGNORE INTO tmpCreatedByUsers VALUES ";
Zotero_DB::bulkInsert($sql2, $toAdd, 50, false, $shardID);
// Join temp table to query
$sql .= "JOIN groupItems GI ON (GI.itemID=I.itemID)
JOIN tmpCreatedByUsers TCBU ON (TCBU.userID=GI.createdByUserID) ";
}
}
break;
}
}
$sql .= "WHERE I.libraryID=? ";
@@ -129,44 +209,27 @@ class Zotero_Items extends Zotero_DataObjects {
$sql .= "AND INo.sourceItemID IS NULL AND IA.sourceItemID IS NULL ";
}
if (!$includeTrashed) {
$sql .= " AND DI.itemID IS NULL ";
}
$keys = array();
// Pass a list of keys, for when the initial search is done via SQL
if (!empty($params['dbkeys'])) {
$keys = $params['dbkeys'];
}
if (!empty($params['itemKey'])) {
if ($keys) {
$keys = array_intersect($keys, explode(',', $params['itemKey']));
}
else {
$keys = explode(',', $params['itemKey']);
}
if (!$keys) {
return array(
'total' => 0,
'items' => array(),
);
}
$sql .= "AND DI.itemID IS NULL ";
}
// Search on title and creators
/*
if (!empty($params['q'])) {
$re = array('$regex' => new MongoRegex("/" . $params['q'] . "/i"));
$query['$or'] = array(
array('title' => $re),
array('creators.firstName' => $re),
array('creators.lastName' => $re),
array('creators.name' => $re)
);
$sql .= "AND (";
$sql .= "value LIKE ? ";
$sqlParams[] = '%' . $params['q'] . '%';
$sql .= "OR title LIKE ? ";
$sqlParams[] = '%' . $params['q'] . '%';
$sql .= "OR firstName LIKE ? ";
$sqlParams[] = '%' . $params['q'] . '%';
$sql .= "OR lastName LIKE ?";
$sqlParams[] = '%' . $params['q'] . '%';
$sql .= ") ";
}
*/
// Tags
//
@@ -180,7 +243,7 @@ class Zotero_Items extends Zotero_DataObjects {
$tagSets = Zotero_API::getSearchParamValues($params, 'tag');
if ($tagSets) {
$sql2 = "SELECT items.key FROM items WHERE 1 ";
$sql2 = "SELECT itemID FROM items WHERE 1 ";
$sqlParams2 = array();
if ($tagSets) {
@@ -220,10 +283,10 @@ class Zotero_Items extends Zotero_DataObjects {
}
}
$tagKeys = Zotero_DB::columnQuery($sql2, $sqlParams2, $shardID);
$tagItems = Zotero_DB::columnQuery($sql2, $sqlParams2, $shardID);
// No matches
if (!$tagKeys) {
if (!$tagItems) {
return array(
'total' => 0,
'items' => array(),
@@ -231,10 +294,10 @@ class Zotero_Items extends Zotero_DataObjects {
}
// Combine with passed keys
if ($keys) {
$keys = array_intersect($keys, $tagKeys);
if ($itemIDs) {
$itemIDs = array_intersect($itemIDs, $tagItems);
// None of the tag matches match the passed keys
if (!$keys) {
if (!$itemIDs) {
return array(
'total' => 0,
'items' => array(),
@@ -242,41 +305,48 @@ class Zotero_Items extends Zotero_DataObjects {
}
}
else {
$keys = $tagKeys;
$itemIDs = $tagItems;
}
}
if ($itemIDs) {
$sql .= "AND I.itemID IN ("
. implode(', ', array_fill(0, sizeOf($itemIDs), '?'))
. ") ";
$sqlParams = array_merge($sqlParams, $itemIDs);
}
if ($keys) {
$sql .= "AND `key` IN ("
. implode(', ', array_fill(0, sizeOf($keys), '?'))
. ")";
. ") ";
$sqlParams = array_merge($sqlParams, $keys);
}
$sql .= "ORDER BY ";
if (!empty($params['order'])) {
/*
if (!$params['emptyFirst'] && $params['order'] == 'creator') {
$sort[$params['order'] . "IsEmpty"] = $dir;
}
*/
switch ($params['order']) {
case 'dateAdded':
case 'dateModified':
$sql .= $params['order'];
$orderSQL = "I." . $params['order'];
break;
case 'title':
$sql .= "(CASE "
. "WHEN I.itemTypeID=1 THEN INo.title"
. " ELSE (SELECT value FROM itemData WHERE itemID=I.itemID AND fieldID IN (110,111,112,113)) "
. " END)";
$orderSQL = "ISF.sortTitle";
break;
case 'creator':
// TODO
$orderSQL = "ISF.creatorSummary";
break;
case 'addedBy':
if ($isGroup) {
$orderSQL = "TCBU.username";
}
else {
$orderSQL = "1";
}
break;
default:
@@ -284,24 +354,31 @@ class Zotero_Items extends Zotero_DataObjects {
if (!$fieldID) {
throw new Exception("Invalid order field '" . $params['order'] . "'");
}
$sql .= "(SELECT value FROM itemData WHERE itemID=I.itemID AND fieldID=?)";
$orderSQL = "(SELECT value FROM itemData WHERE itemID=I.itemID AND fieldID=?)";
if (!$params['emptyFirst']) {
$sqlParams[] = $fieldID;
}
$sqlParams[] = $fieldID;
}
if (!$params['emptyFirst']) {
$sql .= "IFNULL($orderSQL, '') = '', ";
}
$sql .= $orderSQL;
if (!empty($params['sort'])) {
$sql .= " " . $params['sort'];
}
$sql .= ", ";
}
$sql .= "itemID " . (!empty($params['sort']) ? $params['sort'] : "ASC") . " ";
$sql .= "I.itemID " . (!empty($params['sort']) ? $params['sort'] : "ASC") . " ";
if (!empty($params['limit'])) {
$sql .= "LIMIT ?, ?";
$sqlParams[] = $params['start'] ? $params['start'] : 0;
$sqlParams[] = $params['limit'];
}
$itemIDs = Zotero_DB::columnQuery($sql, $sqlParams, $shardID);
if ($itemIDs) {
@@ -309,224 +386,9 @@ class Zotero_Items extends Zotero_DataObjects {
$results['items'] = Zotero_Items::get($libraryID, $itemIDs);
}
return $results;
}
/**
* Convert an array of itemIDs for a given library into an array of keys
*/
public static function idsToKeys($libraryID, $itemIDs) {
if (!$itemIDs) {
return array();
}
$shardID = Zotero_Shards::getByLibraryID($libraryID);
$sql = "CREATE TEMPORARY TABLE tmpIDs (itemID INTEGER UNSIGNED NOT NULL PRIMARY KEY)";
Zotero_DB::query($sql, false, $shardID);
$sql = "INSERT INTO tmpIDs VALUES ";
Zotero_DB::bulkInsert($sql, $itemIDs, 100, false, $shardID);
$sql = "SELECT `key` FROM tmpIDs TI JOIN items I USING (itemID)";
$keys = Zotero_DB::columnQuery($sql, false, $shardID);
if (!$keys) {
$keys = array();
}
Zotero_DB::query("DROP TEMPORARY TABLE tmpIDs", false, $shardID);
return $keys;
}
public static function searchMongo($libraryID, $onlyTopLevel=false, $params=array(), $includeTrashed=false) {
$results = array('items' => array(), 'total' => 0);
$query = array();
$fieldsToReturn = array("_id");
// Filter by libraryID
$query['_id'] = array('$regex' => new MongoRegex("/^$libraryID\//"));
if ($onlyTopLevel) {
$query['parent'] = array('$exists' => false);
}
if (!$includeTrashed) {
$query['deleted'] = array('$ne' => 1);
}
$keys = array();
// Pass a list of keys, for when the initial search is done via SQL
if (!empty($params['dbkeys'])) {
$keys = $params['dbkeys'];
}
if (!empty($params['itemKey'])) {
if ($keys) {
$keys = array_intersect($keys, explode(',', $params['itemKey']));
}
else {
$keys = explode(',', $params['itemKey']);
}
if (!$keys) {
return array(
'total' => 0,
'items' => array(),
);
}
}
// Search on title and creators
if (!empty($params['q'])) {
$re = array('$regex' => new MongoRegex("/" . $params['q'] . "/i"));
$query['$or'] = array(
array('title' => $re),
array('creators.firstName' => $re),
array('creators.lastName' => $re),
array('creators.name' => $re)
);
}
// Tags
//
// ?tag=foo
// ?tag=foo bar // phrase
// ?tag=-foo // negation
// ?tag=\-foo // literal hyphen (only for first character)
// ?tag=foo&tag=bar // AND
// ?tag=foo&tagType=0
// ?tag=foo bar || bar&tagType=0
$tagSets = Zotero_API::getSearchParamValues($params, 'tag');
if ($tagSets) {
$sql = "SELECT items.key FROM items WHERE 1 ";
$sqlParams = array();
if ($tagSets) {
foreach ($tagSets as $set) {
$positives = array();
$negatives = array();
$tagIDs = array();
foreach ($set['values'] as $tag) {
$ids = Zotero_Tags::getIDs($libraryID, $tag);
if (!$ids) {
$ids = array(0);
}
$tagIDs = array_merge($tagIDs, $ids);
}
$tagIDs = array_unique($tagIDs);
if ($set['negation']) {
$negatives = array_merge($negatives, $tagIDs);
}
else {
$positives = array_merge($positives, $tagIDs);
}
if ($positives) {
$sql .= "AND itemID IN (SELECT itemID FROM items JOIN itemTags USING (itemID)
WHERE tagID IN (" . implode(',', array_fill(0, sizeOf($positives), '?')) . ")) ";
$sqlParams = array_merge($sqlParams, $positives);
}
if ($negatives) {
$sql .= "AND itemID NOT IN (SELECT itemID FROM items JOIN itemTags USING (itemID)
WHERE tagID IN (" . implode(',', array_fill(0, sizeOf($negatives), '?')) . ")) ";
$sqlParams = array_merge($sqlParams, $negatives);
}
}
}
$tagKeys = Zotero_DB::columnQuery($sql, $sqlParams, Zotero_Shards::getByLibraryID($libraryID));
// No matches
if (!$tagKeys) {
return array(
'total' => 0,
'items' => array(),
);
}
// Combine with passed keys
if ($keys) {
$keys = array_intersect($keys, $tagKeys);
// None of the tag matches match the passed keys
if (!$keys) {
return array(
'total' => 0,
'items' => array(),
);
}
}
else {
$keys = $tagKeys;
}
}
if ($keys) {
// Add keys to query
array_walk($keys, function (&$key, $index, $prefix) {
$key = $prefix . $key;
}, $libraryID . "/");
if ($query['_id']) {
$query["_id"] = array_merge($query["_id"], array('$in' => $keys));
}
else {
$query["_id"] = array('$in' => $keys);
}
}
// Run query
$cursor = Z_Core::$Mongo->find("searchItems", $query, $fieldsToReturn);
if (!empty($params['order'])) {
$sort = array();
$dir = $params['sort'] == 'desc' ? -1 : 1;
// TEMP: When Mongo supports advanced queries, support emptyFirst
// for fields other than creator
if (!$params['emptyFirst'] && $params['order'] == 'creator') {
$sort[$params['order'] . "IsEmpty"] = $dir;
}
// Use a special field for sorting by title, since we need to
// include display/note titles
if ($params['order'] == 'title') {
$params['order'] = 'sortTitle';
}
if ($params['order'] == 'creator') {
$params['order'] = 'creatorSummary';
}
$sort[$params['order']] = $dir;
$cursor->sort($sort);
}
if (!empty($params['start'])) {
$cursor->skip($params['start']);
}
if (!empty($params['limit'])) {
$cursor->limit($params['limit']);
}
$results['total'] = $cursor->count();
if ($results['total']) {
while ($doc = $cursor->getNext()) {
list($libraryID, $key) = explode('/', $doc['_id']);
$item = Zotero_Items::getByLibraryAndKey($libraryID, $key);
if (!$item) {
Z_Core::logError("Item $libraryID/$key from Mongo not found");
$results['total']--;
continue;
}
$results['items'][] = $item;
}
if (isset($deleteTempTable)) {
$sql = "DROP TEMPORARY TABLE IF EXISTS tmpCreatedByUsers";
Zotero_DB::query($sql, false, $shardID);
}
return $results;
@@ -1461,7 +1323,6 @@ class Zotero_Items extends Zotero_DataObjects {
if ($forceChange) {
$item->setField('dateModified', Zotero_DB::getTransactionTimestamp());
}
Zotero_Index::$queueingEnabled = false;
$item->save($userID);
// Additional steps that have to be performed on a saved object
@@ -1482,7 +1343,6 @@ class Zotero_Items extends Zotero_DataObjects {
$childItem->setSource($item->id);
$childItem->setNote($note->note);
$childItem->save();
Zotero_Index::addItem($childItem);
}
break;
@@ -1500,8 +1360,6 @@ class Zotero_Items extends Zotero_DataObjects {
}
$item->save($userID);
}
Zotero_Index::addItem($item);
Zotero_Index::$queueingEnabled = true;
Zotero_DB::commit();
}
@@ -1798,6 +1656,14 @@ class Zotero_Items extends Zotero_DataObjects {
}
public static function getSortTitle($title) {
if (!$title) {
return '';
}
return mb_substr(preg_replace('/^[\[\(\{\-"\'“‘]([^\]\)\}\-"\'”’]*)[\]\)\}\-"\'”’]?$/u', '$1', $title), 0, Zotero_Notes::$MAX_TITLE_LENGTH);
}
public static function getDataValueCacheKey($hash) {
return 'itemDataValue_' . $hash;
}

12
model/Processor.inc.php
View File

@@ -147,16 +147,4 @@ class Zotero_Error_Processor extends Zotero_Processor {
Zotero_Processors::notifyProcessor($this->mode, $signal, $this->addr, $this->port);
}
}
class Zotero_Index_Processor extends Zotero_Processor {
protected $mode = 'index';
public function __construct() {
$this->port = Z_CONFIG::$PROCESSOR_PORT_INDEX;
}
protected function processFromQueue() {
return Zotero_Index::processFromQueue($this->id);
}
}
?>

26
model/ProcessorDaemon.inc.php
View File

@@ -399,30 +399,4 @@ class Zotero_Error_Processor_Daemon extends Zotero_Processor_Daemon {
Zotero_Sync::purgeErrorProcess($id);
}
}
class Zotero_Index_Processor_Daemon extends Zotero_Processor_Daemon {
protected $mode = 'index';
public function __construct($config=array()) {
$this->port = Z_CONFIG::$PROCESSOR_PORT_INDEX;
parent::__construct($config);
}
public function log($msg) {
Z_Log::log(Z_CONFIG::$PROCESSOR_LOG_TARGET_INDEX, $msg);
}
protected function countQueuedProcesses() {
return Zotero_Index::countQueuedProcesses();
}
protected function getOldProcesses($host=null, $seconds=null) {
return Zotero_Index::getOldProcesses($seconds);
}
protected function removeProcess($id) {
Zotero_Index::removeProcess($id);
}
}
?>

23
model/Sync.inc.php
View File

@@ -1393,7 +1393,6 @@ class Zotero_Sync {
try {
Z_Core::$MC->begin();
Zotero_Index::begin();
Zotero_DB::beginTransaction();
// Mark libraries as updated
@@ -1427,7 +1426,6 @@ class Zotero_Sync {
$creatorObj = Zotero_Creators::convertXMLToCreator($xmlElement);
$creatorObj->save();
$addedLibraryIDs[] = $creatorObj->libraryID;
$addedCreatorDataHashes[] = $creatorObj->creatorDataHash;
}
}
catch (Exception $e) {
@@ -1449,25 +1447,6 @@ class Zotero_Sync {
throw new Exception("libraryID inserted into `creators` not found in `shardLibraries` ($addedLibraryID, $shardID)");
}
}
// creatorDataHash
//
// Check Mongo in chunks to avoid cursor timeouts
$chunks = array_chunk(array_unique($addedCreatorDataHashes), 50);
foreach ($chunks as $chunk) {
$cursor = Z_Core::$Mongo->find("creatorData", array("_id" => array('$in' => $chunk)), array("_id"));
$hashes = array();
while ($cursor->hasNext()) {
$arr = $cursor->getNext();
$hashes[] = $arr['_id'];
}
$added = sizeOf($chunk);
$count = sizeOf($hashes);
if ($count != $added) {
$missing = array_diff($chunk, $hashes);
throw new Exception("creatorDataHashes inserted into `creators` not found in `creatorData` (" . implode(",", $missing) . ") $added $count");
}
}
}
// Add/update items
@@ -1678,7 +1657,6 @@ class Zotero_Sync {
self::removeUploadProcess($processID);
Zotero_Index::commit();
Zotero_DB::commit();
Z_Core::$MC->commit();
@@ -1694,7 +1672,6 @@ class Zotero_Sync {
catch (Exception $e) {
Z_Core::$MC->rollback();
Zotero_DB::rollback(true);
Zotero_Index::rollback();
self::removeUploadProcess($processID);
throw $e;
}

15
processor/index/daemon.php
View File

@@ -1,15 +0,0 @@
<?
if (file_exists('../config')) {
include('../config');
}
if (file_exists('./config')) {
include('./config');
}
set_include_path("../../include");
require("header.inc.php");
require("../../model/ProcessorDaemon.inc.php");
$daemon = new Zotero_Index_Processor_Daemon(!empty($daemonConfig) ? $daemonConfig : array());
$daemon->run();
?>

21
processor/index/processor.php
View File

@@ -1,21 +0,0 @@
<?
error_reporting(E_ALL | E_STRICT);
set_time_limit(120);
if (file_exists('../config')) {
include('../config');
}
if (file_exists('./config')) {
include('./config');
}
set_include_path("../../include");
require("header.inc.php");
require('../../model/Error.inc.php');
require('../../model/Processor.inc.php');
$id = isset($argv[1]) ? $argv[1] : null;
$processor = new Zotero_Index_Processor();
$processor->run($id);
?>