From b3785cb790634d515eff2c72ba726e146a01cffa Mon Sep 17 00:00:00 2001 From: fisharebest Date: Sat, 24 Apr 2010 13:11:21 +0000 Subject: New gedcom importer/uploader/loader --- admin.php | 2 +- editgedcoms.php | 444 ++++++++------ export_gedcom.php | 4 +- import.php | 234 ++++++++ includes/classes/class_menubar.php | 2 +- includes/session.php | 2 +- setup.php | 9 +- uploadgedcom.php | 1169 ------------------------------------ 8 files changed, 498 insertions(+), 1368 deletions(-) create mode 100644 import.php delete mode 100644 uploadgedcom.php diff --git a/admin.php b/admin.php index 98d1adddf1..0d55718206 100644 --- a/admin.php +++ b/admin.php @@ -146,7 +146,7 @@ echo WT_JS_START, 'function showchanges() {window.location.reload();}', WT_JS_EN + href="editgedcoms.php"> diff --git a/editgedcoms.php b/editgedcoms.php index 12fc7ebefd..928e53b1cf 100644 --- a/editgedcoms.php +++ b/editgedcoms.php @@ -5,9 +5,6 @@ * webtrees: Web based Family History software * Copyright (C) 2010 webtrees development team. * - * Derived from PhpGedView - * Copyright (C) 2002 to 2009 PGV Development Team. All rights reserved. - * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -22,219 +19,286 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * - * This Page Is Valid XHTML 1.0 Transitional! > 12 September 2005 - * - * @author PGV Development Team - * @package webtrees - * @subpackage Admin * @version $Id$ */ define('WT_SCRIPT_NAME', 'editgedcoms.php'); require './includes/session.php'; -$all_gedcoms=get_all_gedcoms(); -asort($all_gedcoms); -$action =safe_GET('action', array('delete', 'setdefault')); -$ged =safe_GET('ged', $all_gedcoms); -$default_ged=safe_GET('default_ged', $all_gedcoms); - -/** - * Check if a gedcom file is downloadable over the internet - * - * @author opus27 - * @param string $gedfile gedcom file - * @return mixed $url if file is downloadable, false if not - */ -function check_gedcom_downloadable($gedfile) { - $url = 'http://localhost/'; - if (substr($url,-1,1)!='/') $url .= '/'; - $url .= rawurlencode($gedfile); - @ini_set('user_agent','MSIE 4\.0b2;'); // force a HTTP/1.0 request - @ini_set('default_socket_timeout', '10'); // timeout - $handle = @fopen ($url, 'r'); - if ($handle==false) return false; - // open successfull : now make sure this is a GEDCOM file - $txt = fread ($handle, 80); - fclose($handle); - if (strpos($txt, ' HEAD')==false) return false; - return $url; -} - -//-- make sure that they have admin status before they can use this page -//-- otherwise have them login again +// The gedcom admin page is for gedcom administrators only! if (!WT_USER_GEDCOM_ADMIN) { header('Location: login.php?url=editgedcoms.php'); exit; } -if ($action=='delete') { - delete_gedcom(get_id_from_gedcom($ged)); - // Reload this page, otherwise the page header will still reference the now-deleted gedcom - header('Location: editgedcoms.php'); -} -print_header(i18n::translate('GEDCOM Administration')); -echo '
'; +// Which directory contains our data files? +$INDEX_DIRECTORY=get_site_setting('INDEX_DIRECTORY'); -if (($action=='setdefault') && in_array($default_ged, $all_gedcoms)) { - set_site_setting('DEFAULT_GEDCOM', $default_ged); - $DEFAULT_GEDCOM=$default_ged; -} else { - $DEFAULT_GEDCOM=get_site_setting('DEFAULT_GEDCOM'); +function import_gedcom_file($gedcom_id, $file_name) { + global $TBLPREFIX; + + $fp=fopen($file_name, 'rb'); + WT_DB::exec("START TRANSACTION"); + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_gedcom=?, import_offset=1". + " WHERE gedcom_id=?" + ) + ->bindParam(1, $fp, PDO::PARAM_LOB) + ->bindParam(2, $gedcom_id, PDO::PARAM_INT) + ->execute(); + WT_DB::exec("COMMIT"); + fclose($fp); } -?> -

-
- -'; -if (WT_USER_IS_ADMIN && count($all_gedcoms)>1) { - echo i18n::translate('Default GEDCOM'), ' ', help_link('default_gedcom'); - echo '

'; + break; } -echo'', i18n::translate('Check if GEDCOM files are downloadable'), '', help_link('SECURITY_CHECK_GEDCOM_DOWNLOADABLE'); -// Print table heading -if (WT_USER_IS_ADMIN) { - echo '
'; - echo '', i18n::translate('Add GEDCOM'), ''.help_link('help_addgedcom.php'); - echo ''; - echo '', i18n::translate('Upload GEDCOM'), ''.help_link('help_uploadgedcom.php'); - echo ''; - echo '', i18n::translate('Create a new GEDCOM'), ''.help_link('help_addnewgedcom.php'); - echo '
'; +$gedcoms=WT_DB::prepare( + "SELECT gedcom_id, gedcom_name, import_offset". + " FROM {$TBLPREFIX}gedcom". + " ORDER BY gedcom_name" +)->fetchAll(); + +$all_gedcoms=array(); +foreach ($gedcoms as $gedcom) { + $all_gedcoms[$gedcom->gedcom_id]=$gedcom->gedcom_name; } -echo ''; -$GedCount = 0; - -// Print the table of available GEDCOMs -foreach ($all_gedcoms as $ged_id=>$ged_name) { - if (userGedcomAdmin(WT_USER_ID, $ged_id)) { - // Row 0: Separator line - if ($GedCount!=0) { - echo ''; + +print_header(i18n::translate('GEDCOM administration')); +echo '

', i18n::translate('GEDCOM administration'), '

'; + +// Process GET actions +switch (safe_GET('action')) { +case 'uploadform': +case 'importform': + $gedcom_id=safe_GET('gedcom_id'); + $gedcom_name=get_gedcom_from_id($gedcom_id); + // Check it exists + if (!$gedcom_name) { + break; + } + echo '

', i18n::translate('This will delete all the genealogical data from %s and replace it with data from another GEDCOM.', $gedcom_name), '

'; + echo ''; + echo ''; + if (safe_GET('action')=='uploadform') { + echo ''; + echo ''; + echo '

'; + } else { + echo ''; + $d=opendir($INDEX_DIRECTORY); + $files=array(); + while (($f=readdir($d))!==false) { + if (is_file($INDEX_DIRECTORY.$f) && is_readable($INDEX_DIRECTORY.$f)) { + $fp=fopen($INDEX_DIRECTORY.$f, 'rb'); + $header=fread($fp, 64); + fclose($fp); + if (preg_match('/^('.WT_UTF8_BOM.')?0 *HEAD/', $header)) { + $files[]=$f; + } + } } - $GedCount++; - - // Row 1: Heading - echo ''; - echo ''; - - - // Row 2: GEDCOM file name & functions - echo ''; - - echo ''; + } + echo ''; + echo ''; + print_footer(); + exit; +} + + +// List the gedcoms available to this user +foreach ($gedcoms as $gedcom) { + if (userGedcomAdmin(WT_USER_ID, $gedcom->gedcom_id)) { - echo '


', i18n::translate('GEDCOM title'), ''; - if ($DEFAULT_GEDCOM==$ged_name) echo ''; - echo PrintReady(get_gedcom_setting($ged_id, 'title')).'  '; - if ($TEXT_DIRECTION=='rtl') echo getRLM(), '('.$ged_id.')', getRLM(); - else echo getLRM(), '(', $ged_id, ')', getLRM(); - if ($DEFAULT_GEDCOM==$ged_name) echo ''; - echo '  ', i18n::translate('Upload Replacement'), '
'; // Column 1 (row legend) - echo i18n::translate('GEDCOM file'); - echo ''; // Column 2 (file name & notices) - if (file_exists(get_gedcom_setting($ged_id, 'path'))) { - if ($TEXT_DIRECTION=='ltr') echo get_gedcom_setting($ged_id, 'path'), ' ('; - else echo getLRM(), get_gedcom_setting($ged_id, 'path'), ' ', getRLM(), '('; - printf('%.2fKB', (filesize(get_gedcom_setting($ged_id, 'path'))/1024)); - echo ')'; - /** deactivate [ 1573749 ] - * -- activating based on a request parameter instead of a config parameter - */ - if(!empty($_REQUEST['check_download'])){ - $url = check_gedcom_downloadable(get_gedcom_setting($ged_id, 'path')); - if ($url!==false) { - echo '
', i18n::translate('This GEDCOM file is downloadable over the internet!
Please see the SECURITY section of the readme.txt file to fix this problem'), ' :'; - echo '
', $url, ''; + if ($files) { + echo $INDEX_DIRECTORY, ''; + echo '

'; + } else { + echo '

', i18n::translate('No GEDCOM files found. You need to copy files to the %s directory on your server.', $INDEX_DIRECTORY); } - else echo '', i18n::translate('File not found.'), ''; - echo '

'; // Column 3 (Import action) - echo '', i18n::translate('Import'), ''; - if (!get_gedcom_setting($ged_id, 'imported')) { - echo '
', i18n::translate('This GEDCOM has not yet been imported.'), ''; + echo + '', + ''; - - echo ''; - - echo ''; - - echo ''; - - echo ''; - - echo ''; - - - // Row 3: Configuration file - echo ''; - echo ''; - - echo ''; - - echo ''; - - echo ''; - echo ''; - - // Row 4: Privacy File - echo ''; - echo ''; - - echo ''; - - echo ''; - - echo ''; - echo ''; + echo + '
', i18n::translate('GEDCOM name'), + 'gedcom_name), '">', htmlspecialchars($gedcom->gedcom_name), ' - ', + htmlspecialchars(get_gedcom_setting($gedcom->gedcom_id, 'title')), '', + '
', i18n::translate('GEDCOM administration'), + ''; + + // The third row shows an optional progress bar and a list of maintenance options + if ($gedcom->import_offset>0) { + echo + '
', + WT_JS_START, + '$("#import', $gedcom->gedcom_id, '").load("import.php?gedcom_id=', $gedcom->gedcom_id, '")', + WT_JS_END; } - echo '  		'; // Column 4 (Export action) - echo '', i18n::translate('Export'), ''; - echo ''; // Column 5 (Delete action) - echo '", i18n::translate('Delete'), ''; - echo '  '; - echo ''; // Column 6 (Download action) - echo '', i18n::translate('Download'), ''; - echo '  '; - echo ''; // Column 7 (Check action) - echo '', i18n::translate('Check'), ''; - echo '  '; - echo '
'; // Column 1 (row legend) - echo i18n::translate('Configuration file'); - echo ''; // Column 2 (file name & notices) - echo getLRM() . get_config_file($ged_id); - echo ''; // Column 3 (Edit action) - echo '', i18n::translate('Edit'), ''; - echo ''; // Columns 4-7 (blank) - echo ' '; - echo '
'; // Column 1 (row legend) - echo i18n::translate('Privacy file'); - echo ''; // Column 2 (file name & notices) - echo getLRM(), get_privacy_file($ged_id); - echo ''; // Column 3 (Edit action) - echo '', i18n::translate('Edit'), ''; - echo ''; // Columns 4-7 (blank) - echo ' '; - echo '
', + // configuration + '', + // privacy + '', + // export + '', + // import + '', + // download + '', + // upload + '', + // delete + '
', i18n::translate('Configuration'), '', + help_link('gedcom_configfile'), + '', i18n::translate('Privacy'), '', + help_link('edit_privacy'), + 'gedcom_name}"), '\', \'_blank\',\'left=50,top=50,width=500,height=500,resizable=1,scrollbars=1\');">', i18n::translate('Export'), '', + help_link('export_gedcom.php'), + '', i18n::translate('Import'), '', + help_link('import_gedcom.php'), + '', i18n::translate('Download'), '', + help_link('ownload_gedcom'), + '', i18n::translate('Upload'), '', + help_link(''), + '', i18n::translate('Delete'), '', + help_link('delete_gedcom'), + '

'; } } -echo '
'; -require get_config_file(WT_GED_ID); +// Options for creating new gedcoms and setting defaults +if (WT_USER_IS_ADMIN) { + echo + '
', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '
', i18n::translate('Default GEDCOM'), help_link('default_gedcom'), '', i18n::translate('Add a new GEDCOM'), help_link('help_addgedcom.php'), '', i18n::translate('Upload a new GEDCOM'), help_link('help_uploadgedcom.php'), '', i18n::translate('Create a new GEDCOM'), help_link('help_addnewgedcom.php'), '
', + '
', + '', + '', + '
', + '
', + $INDEX_DIRECTORY, + '', + '', + '
', + '		', + '
', + '', + '', + '
', + '		', + '
', + '', + '', + ' ', + '
', + '

'; +} print_footer(); -?> diff --git a/export_gedcom.php b/export_gedcom.php index ea950751c6..e4f62ca8a0 100644 --- a/export_gedcom.php +++ b/export_gedcom.php @@ -52,7 +52,7 @@ print_simple_header(i18n::translate('Export')); if ($export) { $ged_id = get_id_from_gedcom($export); - $filename = get_gedcom_setting($ged_id, 'path'); + $filename = get_site_setting('INDEX_DIRECTORY').$export; echo '

', i18n::translate('Export'), '

'; echo '

', htmlspecialchars(filename_decode($export)), ' => ', $filename, '

'; flush(); @@ -71,7 +71,7 @@ if ($export) { $end = microtime(true); fclose($gedout); - unlink($filename); + @unlink($filename); rename($filename.'.tmp', $filename); $stat = stat($filename); echo sprintf('

%d bytes, %0.3f seconds

', $stat['size'], $end-$start); diff --git a/import.php b/import.php new file mode 100644 index 0000000000..5a7855478a --- /dev/null +++ b/import.php @@ -0,0 +1,234 @@ +, where NNN is the gedcom ID. + * It will then call import.php to load the div's contents using AJAX. + * + * We start importing at position wt_gedcom.import_offset and continue + * for a couple of seconds. When we've finished we set import_offset to + * zero to indicate success. + * + * webtrees: Web based Family History software + * Copyright (C) 2010 webtrees development team. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * @version $Id$ + */ + +define('WT_SCRIPT_NAME', 'import.php'); +require './includes/session.php'; + +require_once WT_ROOT.'includes/functions/functions_import.php'; + +// Don't use ged=XX as we want to be able to run without changing the current gedcom. +// This will let us load several gedcoms together, or to edit one while loading another. +$gedcom_id=safe_GET('gedcom_id'); + +if (!userGedcomAdmin(WT_USER_ID, $gedcom_id)) { + header('HTTP/1.0 403 Forbidden'); + exit; +} + +// Don't allow the user to cancel the request. We do not want +// to be left with an incomplete transaction, as this could cause a +// timeout error in another session. +ignore_user_abort(true); + +// Run in a transaction, and make sure we are the only ones importing this gedcom +WT_DB::exec("START TRANSACTION"); + +// What is the current import status? +$row=WT_DB::prepare( + "SELECT import_offset, LENGTH(import_gedcom) AS import_total FROM {$TBLPREFIX}gedcom WHERE gedcom_id=? FOR UPDATE" +)->execute(array($gedcom_id))->fetchOneRow(); + +if (!$row) { + // No such gedcom? Deleted in another session? die quietly + WT_DB::exec("COMMIT"); + exit; +} + +header('Content-type: text/html; charset=UTF-8'); + +if ($row->import_offset==0 || $row->import_total==0) { + // Finished? Show the maintenance links, similar to editgedcoms.php + WT_DB::exec("COMMIT"); + echo "DONE"; + exit; +} + +// Calculate progress so far +$percent=100*(($row->import_offset-1) / $row->import_total); +if ($row->import_offset==1) { + $status=i18n::translate('Initialising ...'); +} else { + $status=sprintf('%.1f%%', $percent); +} + +echo + '
', htmlspecialchars($status), '
', + WT_JS_START, + '$("#progressbar").progressbar({value: ', round($percent, 1), '});', + WT_JS_END, +flush(); + +// Run for one second. This keeps the resource requirements low. +for ($end_time=microtime(true)+1.0; microtime(true)<$end_time; ) { + // If we are at the start position, do some tidying up + if ($row->import_offset==1) { + // Delete any existing genealogical data + empty_database($gedcom_id, false); + set_gedcom_setting($gedcom_id, 'imported', false); + // Remove any byte-order-mark + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_gedcom=TRIM(LEADING ? FROM import_gedcom)". + " WHERE gedcom_id=?" + )->execute(array(WT_UTF8_BOM, $gedcom_id)); + // Convert line endings. Don't convert \r\n - it is very slow. Just deal + // with empty records later. + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_gedcom=REPLACE(import_gedcom, '\r', '\n')". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id)); + // Fetch the header record + $data=WT_DB::prepare( + "SELECT LEFT(import_gedcom, CASE LOCATE('\n0', import_gedcom, 2) WHEN 0 THEN LENGTH(import_gedcom) ELSE LOCATE('\n0', import_gedcom, 2) END)". + " FROM {$TBLPREFIX}gedcom". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id))->fetchOne(); + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_offset=?". + " WHERE gedcom_id=?" + )->execute(array(strlen($data)+1, $gedcom_id)); + if (substr($data, 0, 6)!='0 HEAD') { + WT_DB::exec("ROLLBACK"); + echo i18n::translate('Invalid GEDCOM file - no header record found.'); + exit; + } + // What character set is this? Need to convert it to UTF8 + if (preg_match('/\n1\s*CHAR(ACTER)\s+(.+)/', $data, $match)) { + $charset=strtoupper($match[1]); + } else { + $charset='ASCII'; + } + // MySQL supports a wide range of collation conversions. These are ones that + // have been encountered "in the wild". + switch ($charset) { + case 'ANSI': + case 'ASCII': + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_gedcom=CONVERT(CONVERT(import_gedcom USING ascii) USING utf8)". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id)); + break; + case 'IBMPC': // IBMPC could be anything. Mostly it means CP850. + case 'CP437': + case 'CP850': + // CP850 has extra letters with diacritics to replace box-drawing chars in CP437. + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_gedcom=CONVERT(CONVERT(import_gedcom USING cp850) USING utf8)". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id)); + break; + case 'MACINTOSH': + // Convert from MAC Roman to UTF8. + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_gedcom=CONVERT(CONVERT(import_gedcom USING macroman) USING utf8)". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id)); + break; + case 'UTF8': + case 'UTF-8': + // Already UTF-8 so nothing to do! + break; + case 'ANSEL': + // TODO: fisharebest has written a mysql stored procedure that converts ANSEL to UTF-8 + default: + echo + WT_JS_START, + 'alert(', htmlspecialchars(i18n::translate('Error: cannot convert GEDCOM file from %s encoding to UTF-8 encoding.', $charset)), ');', + WT_JS_END; + break; + } + $data=preg_replace('/\n1 CHAR.*(\n[2-9].+)*/', '', $data)."\n1 CHAR UTF-8"; + import_record(trim($data), $gedcom_id, false); + } else { + // Fetch the next block of data. At least 64KB, and ending on a record boundary. + $data=WT_DB::prepare( + "SELECT". + " CASE LOCATE('\n0', import_gedcom, import_offset+65536)". + " WHEN 0 THEN SUBSTR(import_gedcom FROM import_offset)". + " ELSE SUBSTR(import_gedcom FROM import_offset FOR LOCATE('\n0', import_gedcom, import_offset+65536)-import_offset)". + " END". + " FROM {$TBLPREFIX}gedcom". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id))->fetchOne(); + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_offset=import_offset+?". + " WHERE gedcom_id=?" + )->execute(array(strlen($data), $gedcom_id)); + echo WT_JS_START, WT_JS_END; + foreach (preg_split('/\n(?=0)/', $data) as $rec) { + if ($rec) { + import_record(trim($rec), $gedcom_id, false); + } + } + // Recalculate progress and update status bar. + /* This isn't working. These JS statements don't run until the script has finished. :-( + $row->import_offset+=strlen($data); + $percent=100*($row->import_offset / $row->import_total); + $status=sprintf('%.1f%%', $percent); + echo + WT_JS_START, + '$("#progress', $gedcom_id, '").attr("width", "', round($percent, 1), '%");', + '$("#status', $gedcom_id, '").html("', htmlspecialchars($status), '");', + WT_JS_END; + flush(); + */ + } +} + +if ($row->import_offset>$row->import_total) { + // Done + set_gedcom_setting($gedcom_id, 'imported', true); + WT_DB::prepare( + "UPDATE {$TBLPREFIX}gedcom". + " SET import_offset=0". + " WHERE gedcom_id=?" + )->execute(array($gedcom_id)); + echo + WT_JS_START, + '$("#import', $gedcom_id, '").toggle();', + '$("#actions', $gedcom_id, '").toggle();', + WT_JS_END; +} else { + // Reload..... + echo + WT_JS_START, + '$("#import', $gedcom_id, '").load("import.php?gedcom_id=', $gedcom_id, '");', + WT_JS_END; +} + +WT_DB::exec("COMMIT"); diff --git a/includes/classes/class_menubar.php b/includes/classes/class_menubar.php index 617b32d459..04061264a9 100644 --- a/includes/classes/class_menubar.php +++ b/includes/classes/class_menubar.php @@ -153,7 +153,7 @@ class MenuBar $submenu->addClass("submenuitem$ff", "submenuitem_hover$ff", "", "icon_small_admin"); $menu->addSubmenu($submenu); //-- manage_gedcoms submenu - $submenu = new Menu(i18n::translate('Manage GEDCOMs and edit Privacy'), "editgedcoms.php"); + $submenu = new Menu(i18n::translate('GEDCOM administration'), "editgedcoms.php"); if (!empty($WT_IMAGES["admin"]["small"])) $submenu->addIcon($WT_IMAGE_DIR."/".$WT_IMAGES["admin"]["small"]); $submenu->addClass("submenuitem$ff", "submenuitem_hover$ff", "", "icon_small_admin"); diff --git a/includes/session.php b/includes/session.php index 0c68fb548c..9549152f01 100644 --- a/includes/session.php +++ b/includes/session.php @@ -424,7 +424,7 @@ if (isset($SHOW_CONTEXT_HELP) && $show_context_help==='no') $_SESSION['show_cont if (!isset($USE_THUMBS_MAIN)) $USE_THUMBS_MAIN = false; if (WT_SCRIPT_NAME!='help_text.php') { - if (!get_gedcom_setting(WT_GED_ID, 'imported') && !in_array(WT_SCRIPT_NAME, array('editconfig_gedcom.php', 'help_text.php', 'editgedcoms.php', 'downloadgedcom.php', 'uploadgedcom.php', 'login.php', 'siteconfig.php', 'admin.php', 'addnewgedcom.php', 'validategedcom.php', 'addmedia.php', 'importgedcom.php', 'client.php', 'edit_privacy.php', 'gedcheck.php', 'useradmin.php', 'export_gedcom.php', 'edit_changes.php'))) { + if (!get_gedcom_setting(WT_GED_ID, 'imported') && !in_array(WT_SCRIPT_NAME, array('editconfig_gedcom.php', 'help_text.php', 'editgedcoms.php', 'downloadgedcom.php', 'uploadgedcom.php', 'login.php', 'siteconfig.php', 'admin.php', 'addnewgedcom.php', 'validategedcom.php', 'addmedia.php', 'importgedcom.php', 'client.php', 'edit_privacy.php', 'gedcheck.php', 'useradmin.php', 'export_gedcom.php', 'edit_changes.php', 'import.php'))) { header('Location: editgedcoms.php'); exit; } diff --git a/setup.php b/setup.php index efa7f209a9..818d6732fc 100644 --- a/setup.php +++ b/setup.php @@ -589,8 +589,10 @@ try { $TBLPREFIX=$_POST['tblpfx']; $dbh->exec( "CREATE TABLE IF NOT EXISTS {$TBLPREFIX}gedcom (". - " gedcom_id INTEGER AUTO_INCREMENT NOT NULL,". - " gedcom_name VARCHAR(255) NOT NULL,". + " gedcom_id INTEGER AUTO_INCREMENT NOT NULL,". + " gedcom_name VARCHAR(255) NOT NULL,". + " import_gedcom LONGBLOB NOT NULL DEFAULT '',". + " import_offset INTEGER UNSIGNED NOT NULL DEFAULT 1,". " PRIMARY KEY (gedcom_id),". " UNIQUE KEY ux1 (gedcom_name)". ") COLLATE utf8_unicode_ci ENGINE=InnoDB" @@ -977,7 +979,6 @@ try { $dbh->exec( "INSERT IGNORE INTO {$TBLPREFIX}site_setting (setting_name, setting_value) VALUES ". "('WT_SCHEMA_VERSION', '1'),". - "('DEFAULT_GEDCOM', 'default.ged'),". "('INDEX_DIRECTORY', 'data/'),". "('AUTHENTICATION_MODULE', 'includes/authentication.php'),". "('STORE_MESSAGES', '1'),". @@ -988,7 +989,7 @@ try { "('SESSION_SAVE_PATH', ''),". "('SESSION_TIME', '7200'),". "('SERVER_URL', ''),". - "('LOGIN_URL', ''),". + "('LOGIN_URL', 'login.php'),". "('MAX_VIEWS', '20'),". "('MAX_VIEW_TIME', '1'),". "('MEMORY_LIMIT', '".addcslashes($_POST['maxmem'], "'")."M'),". diff --git a/uploadgedcom.php b/uploadgedcom.php deleted file mode 100644 index 761b032f1c..0000000000 --- a/uploadgedcom.php +++ /dev/null @@ -1,1169 +0,0 @@ - 12 September 2005 - * - * @author PGV Development Team - * @package webtrees - * @subpackage Admin - * @version $Id$ - */ - -// TODO: Progress bars don't show until or -// TODO: Upload ZIP support alternative path and name - -// NOTE: $GEDFILENAME = The filename of the uploaded GEDCOM -// NOTE: $action = Which form we should present -// NOTE: $check = Which check to be performed -// NOTE: $timelimit = The time limit for the import process -// NOTE: $cleanup = If set to yes, the GEDCOM contains invalid tags -// NOTE: $no_upload = When the user cancelled, we want to restore the original settings -// NOTE: $path = The path to the GEDCOM file -// NOTE: $continue = When the user decided to move on to the next step -// NOTE: $import_existing = See if we are just importing an existing GEDCOM -// NOTE: $replace_gedcom = When uploading a GEDCOM, user will be asked to replace an existing one. If yes, overwrite -// NOTE: $bakfile = Name and path of the backupfile, this file is created if a file with the same name exists - -define('WT_SCRIPT_NAME', 'uploadgedcom.php'); -require './includes/session.php'; -require_once WT_ROOT.'includes/functions/functions_import.php'; -require_once WT_ROOT.'includes/functions/functions_export.php'; - -if (!WT_USER_GEDCOM_ADMIN) { - header("Location: login.php?url=uploadgedcom.php"); - exit; -} - -// editconfig.php and uploadgedcom.php make extensive use of -// import_request_variables and are heavily inter-dependent. -@import_request_variables('cgp'); - -@ini_set('zlib.output_compression','0'); - -if (empty ($action)) $action = "upload_form"; -if (!isset ($path)) $path = ""; -if (!isset ($check)) $check = ""; -if (!isset ($error)) $error = ""; -if (!isset ($verify)) $verify = ""; -if (!isset ($import)) $import = false; -if (!isset ($bakfile)) $bakfile = ""; -if (!isset ($cleanup_needed)) $cleanup_needed = false; -if (!isset ($ok)) $ok = false; -if (!isset ($startimport)) $startimport = false; -if (!isset ($timelimit)) $timelimit = get_site_setting('MAX_EXECUTION_TIME'); -if (!isset ($importtime)) $importtime = 0; -if (!isset ($no_upload)) $no_upload = false; -if (!isset ($override)) $override = false; -if ($no_upload == "cancel_upload" || $override == "no") $check = "cancel_upload"; -if (!isset ($exists)) $exists = false; -if (!isset ($config_gedcom)) $config_gedcom = ""; -if (!isset ($continue)) $continue = false; -if (!isset ($import_existing)) $import_existing = false; -if (!isset($utf8convert)) $utf8convert = "no"; -if (isset($_REQUEST['keepmedia']) && $_REQUEST['keepmedia']=='yes') $keepmedia=true; -else $keepmedia = false; - -// NOTE: GEDCOM was uploaded -if ($check == "upload") { - $verify = "verify_gedcom"; - $ok = true; -} -// NOTE: GEDCOM was added -else if ($check == "add") { - $verify = "verify_gedcom"; - $ok = true; -} else if ($check == "add_new") { - if (((!file_exists($INDEX_DIRECTORY.$GEDFILENAME)) && !file_exists($path.$GEDFILENAME)) || $override == "yes") { - if ($path != "") $fp = fopen($path.$GEDFILENAME, "wb"); - else $fp = fopen($INDEX_DIRECTORY.$GEDFILENAME, "wb"); - if ($fp) { - $newgedcom = gedcom_header($GEDFILENAME). - "0 @I1@ INDI\n". - "1 NAME Given Names /Surname/\n". - "1 SEX M\n". - "1 BIRT\n". - "2 DATE 01 JAN 1850\n". - "2 PLAC Click edit and change me\n". - "0 TRLR\n"; - $newgedcom = preg_replace('/[\r\n]+/', WT_EOL, $newgedcom); - fwrite($fp, $newgedcom); - fclose($fp); - $logline = AddToLog($GEDFILENAME." updated", 'config'); - $verify = "validate_form"; - $exists = true; - // NOTE: Go straight to import, no other settings needed - $xreftype = "NA"; - $utf8convert = "no"; - $ged = $GEDFILENAME; - $startimport = "true"; - //-- set the current GEDCOM to be this new GEDCOM file - //-- so that the import can proceed correctly - $GEDCOM = $GEDFILENAME; - $FILE = $GEDCOM; - } - } else { - if ($path != "") - $fp = fopen($path.$GEDFILENAME.".bak", "wb"); - else - $fp = fopen($INDEX_DIRECTORY.$GEDFILENAME.".bak", "wb"); - if ($fp) { - $newgedcom = gedcom_header($GEDFILENAME). - "0 @I1@ INDI\n". - "1 NAME Given Names /Surname/\n". - "1 SEX M\n". - "1 BIRT\n". - "2 DATE 01 JAN 1850\n". - "2 PLAC Click edit and change me\n". - "0 TRLR\n"; - $newgedcom = preg_replace('/[\r\n]+/', WT_EOL, $newgedcom); - fwrite($fp, $newgedcom); - fclose($fp); - if ($path != "") - $bakfile = $path.$GEDFILENAME.".bak"; - else - $bakfile = $INDEX_DIRECTORY.$GEDFILENAME.".bak"; - $ok = false; - $verify = "verify_gedcom"; - $exists = true; - } - } -} else -if ($check == "cancel_upload") { - if ($exists) { - delete_gedcom(get_id_from_gedcom($GEDFILENAME)); - if ($action == "add_new_form") - @ unlink($INDEX_DIRECTORY.$GEDFILENAME); - } - // NOTE: Cleanup everything no longer needed - if (isset($bakfile) && file_exists($bakfile)) unlink($bakfile); - $verify = ""; - unset($GEDFILENAME); - $startimport = ""; - $import = false; - $cleanup_needed = false; - $noupload = true; - header("Location: editgedcoms.php"); -} - -if ($cleanup_needed == "cleanup_needed" && $continue == i18n::translate('Continue')) { - require_once WT_ROOT.'includes/functions/functions_tools.php'; - - $filechanged = false; - if (file_is_writeable(get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path')) && (file_exists(get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path')))) { - $l_BOMcleanup = false; - $l_headcleanup = false; - $l_macfilecleanup = false; - $l_lineendingscleanup = false; - $l_placecleanup = false; - $l_datecleanup = false; - $l_isansi = false; - $fp = fopen(get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path'), "rb"); - $fw = fopen($INDEX_DIRECTORY."/".$GEDFILENAME.".bak", "wb"); - //-- read the gedcom and test it in 8KB chunks - while (!feof($fp)) { - $fcontents = fread($fp, 1024 * 8); - $lineend = "\n"; - if (need_macfile_cleanup()) { - $l_macfilecleanup = true; - $lineend = "\r"; - } - - //-- read ahead until the next line break - $byte = ""; - while ((!feof($fp)) && ($byte != $lineend)) { - $byte = fread($fp, 1); - $fcontents .= $byte; - } - - if (!$l_BOMcleanup && need_BOM_cleanup()) { - BOM_cleanup(); - $l_BOMcleanup = true; - } - - if (!$l_headcleanup && need_head_cleanup()) { - head_cleanup(); - $l_headcleanup = true; - } - - if ($l_macfilecleanup) { - macfile_cleanup(); - } - - if (isset ($_POST["cleanup_places"]) && $_POST["cleanup_places"] == "YES") { - if (($sample = need_place_cleanup()) !== false) { - $l_placecleanup = true; - place_cleanup(); - } - } - - if (line_endings_cleanup()) { - $filechanged = true; - } - - if (isset ($_POST["datetype"])) { - $filechanged = true; - //month first - date_cleanup($_POST["datetype"]); - } - /** - if($_POST["xreftype"]!="NA") { - $filechanged=true; - xref_change($_POST["xreftype"]); - } - **/ - if (isset ($_POST["utf8convert"]) == "YES") { - $filechanged = true; - convert_ansi_utf8(); - } - fwrite($fw, $fcontents); - } - fclose($fp); - fclose($fw); - copy($INDEX_DIRECTORY."/".$GEDFILENAME.".bak", get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path')); - $cleanup_needed = false; - $import = "true"; - } else { - $error = i18n::translate('The GEDCOM file, %s, is not writable. Please check attributes and access rights.', $GEDFILENAME); - } -} - -// NOTE: Change header depending on action -if ($action == "upload_form") { - print_header(i18n::translate('Upload GEDCOM')); -} elseif ($action == "add_form") { - print_header(i18n::translate('Add GEDCOM')); -} elseif ($action == "add_new_form") { - print_header(i18n::translate('Create a new GEDCOM')); -} else { - print_header(i18n::translate('Import')); -} - -// NOTE: Print form header -echo "
"; - -// NOTE: Print table header -echo "\n"; - -// NOTE: Add GEDCOM form -if ($action == "add_form") { - echo ""; - echo " -"; - echo ""; -} -// NOTE: Add new GEDCOM form -elseif ($action == "add_new_form") { - echo ""; - echo ""; -} - -if ($verify == "verify_gedcom") { - // NOTE: Check if GEDCOM has been imported into DB - $all_record_counts=count_all_records(get_id_from_gedcom($GEDFILENAME)); - $totalRecords = 0; - foreach ($all_record_counts as $recordCount) { - $totalRecords += $recordCount; - } - $imported = !empty($totalRecords); - if ($imported || (!empty($bakfile) && file_exists($bakfile))) { - // NOTE: If GEDCOM exists show warning - print ""; - print ""; - print " -"; - print ""; -} - -if ($startimport == "true") { - set_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'imported', false); - - if (isset ($exectime)) { - $oldtime = time() - $exectime; - $skip_table = 0; - } else - $oldtime = time(); - - /** - * function that sets up the html required to run the progress bar - * @param long $FILE_SIZE the size of the file - */ - function setup_progress_bar($FILE_SIZE) { - global $ged, $timelimit; - ?> - -
"; - echo "\"\""; - echo help_link('add_gedcom'); - echo " "; - if ($import_existing) { - echo i18n::translate('Import'); - } else { - echo i18n::translate('Add GEDCOM'); - } - echo ""; - echo "
"; - echo "
"; - ?> - - - - - "; - } - ?> - - - - -
"; - echo "", $error, "\n"; - echo "
- /> -
-
-
"; - echo "\"\""; - echo " ", i18n::translate('Upload GEDCOM'), ""; - echo help_link('upload_gedcom'); - echo "
"; - echo "
"; -?> - - - -", $error, "
\n"; - echo i18n::translate('This error probably means that the file you tried to upload exceeded the limit set by your host. The default limit in PHP is 2MB. You can contact your host\'s Support group to have them increase the limit in the php.ini file, or you can upload the file using FTP. Use the Add GEDCOM page to add a GEDCOM file you have uploaded using FTP.');; - echo "
\n"; -} -?> - - - - -
- -"; - if (!$filesize = ini_get('upload_max_filesize')) { - $filesize = "2M"; - } - echo " ( ", i18n::translate('Maximum upload size: '), " $filesize )"; -} -?> -
-"; -echo "
"; - echo "\"\""; - echo " ", i18n::translate('Create a new GEDCOM'), ""; - echo help_link('add_gedcom_instructions'); - echo "
"; - echo "
"; -?> - - - -"; -} -?> - - - - -
"; - echo "", $error, "\n"; - echo "
- - />
-"; -echo "
"; - print "\"\""; - print " ".i18n::translate('Verify GEDCOM').""; - print help_link('verify_gedcom'); - print "
"; - print "
"; - print "\n"; - print ""; - // NOTE: Check for existing changes - $changes=WT_DB::prepare("SELECT 1 FROM {$TBLPREFIX}change WHERE status='pending' AND gedcom_id=?")->execute(array(get_id_from_gedcom($GEDFILENAME)))->fetchOne(); - if ($changes) { - echo i18n::translate('The current GEDCOM has changes pending review. If you continue this Import, these pending changes will be discarded. You should review the pending changes before continuing the Import.'); - echo "

"; - } - print ""; - //-- check if there are media in the DB already - if (array_key_exists('OBJE', $all_record_counts)) { - ?> - - - - -
"; - ?> - - - - - - - - ".i18n::translate('A GEDCOM with this file name has already been imported into the database.')."

"; - print "".i18n::translate('This GEDCOM file is not synchronized with the database. It may not contain the latest version of your data. To re-import from the database rather than the file, you should download and re-upload.')."

"; - } - if ($bakfile != "") print i18n::translate('A GEDCOM file with the same name has been found. If you choose to continue, the old GEDCOM file will be replaced with the file that you uploaded and the Import process will begin again. If you choose to cancel, the old GEDCOM will remain unchanged.')."
".i18n::translate('Do you want to erase the old data and replace it with this new data?')."\n"; - print "
- - -
"; - print "
"; - } else { - $verify = "validate_form"; - } -} - -if ($verify == "validate_form") { - print "
"; - print "\"\""; - print " ".i18n::translate('Validate GEDCOM').""; - print help_link('validate_gedcom'); - print "
"; - print "
"; - print "\n"; - print ""; - $import = true; - } else { - $cleanup_needed = true; - print ""; - if (!file_is_writeable(get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path')) && (file_exists(get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path')))) { - print "".i18n::translate('The GEDCOM file, %s, is not writable. Please check attributes and access rights.', $GEDCOM)."\n"; - print ""; - } - // NOTE: Check for BOM cleanup - if ($l_BOMcleanup) { - print ""; - } - // NOTE: Check for head cleanup - if ($l_headcleanup) { - print ""; - } - // NOTE: Check for mac file cleanup - if ($l_macfilecleanup) { - print ""; - } - // NOTE: Check for line endings cleanup - if ($l_lineendingscleanup) { - print ""; - } - // NOTE: Check for place cleanup - if ($l_placecleanup) { - print ""; - } - // NOTE: Check for date cleanup - if ($l_datecleanup) { - print ""; - } - // NOTE: Check for ansi encoding - if ($l_isansi) { - print "
".i18n::translate('Performing GEDCOM validation...')."
"; - if (!empty ($error)) - print "$error\n"; - - if ($import != true) { - require_once WT_ROOT.'includes/functions/functions_tools.php'; - - $l_BOMcleanup = false; - $l_headcleanup = false; - $l_macfilecleanup = false; - $l_lineendingscleanup = false; - $l_placecleanup = false; - $l_datecleanup = false; - $l_isansi = false; - $fp = fopen(get_gedcom_setting(get_id_from_gedcom($GEDFILENAME), 'path'), "r"); - - // TODO - there are two problems with this next block of code. Firstly, by - // checking the file in chunks (rather than complete records), we won't spot - // any problem that spans chunk boundaries. Secondly, the date checking - // stops after the first error. If the first error is not an ambiguous date - // then we won't ask the user to choose between DMY and YMD. - - //-- read the gedcom and test it in 8KB chunks - while ($fp && !feof($fp)) { - $fcontents = fread($fp, 1024 * 8); - if (!$l_BOMcleanup && need_BOM_cleanup()) $l_BOMcleanup = true; - if (!$l_headcleanup && need_head_cleanup()) $l_headcleanup = true; - if (!$l_macfilecleanup && need_macfile_cleanup()) $l_macfilecleanup = true; - if (!$l_lineendingscleanup && need_line_endings_cleanup()) $l_lineendingscleanup = true; - if (!$l_placecleanup && ($placesample = need_place_cleanup()) !== false) $l_placecleanup = true; - if (!$l_datecleanup && ($datesample = need_date_cleanup()) !== false) $l_datecleanup = true; - if (!$l_isansi && is_ansi()) $l_isansi = true; - } - fclose($fp); - - $cleanup_needed = false; - if (!$l_datecleanup && !$l_isansi && !$l_BOMcleanup && !$l_headcleanup && !$l_macfilecleanup && !$l_placecleanup && !$l_lineendingscleanup) { - print i18n::translate('Valid GEDCOM detected. No cleanup required.'); - print "
"; - print "".i18n::translate('A Byte Order Mark (BOM) was detected at the beginning of the file. On cleanup, this special code will be removed.')."\n"; - print help_link('BOM_detected'); - print "
"; - print "".i18n::translate('Detected lines before the GEDCOM header 0 HEAD. On cleanup, these lines will be removed.')."\n"; - print help_link('invalid_header'); - print "
"; - print "".i18n::translate('Macintosh file detected. On cleanup your file will be converted to a DOS file.')."\n"; - print help_link('macfile_detected'); - print "
"; - print "".i18n::translate('Empty lines were detected in your GEDCOM file. On cleanup, these empty lines will be removed.')."\n"; - print help_link('empty_lines_detected'); - print "
"; - print "\n"; - print ""; - print ""; - print "
"; - print "".i18n::translate('Invalid place encodings were detected. These errors should be fixed.')."\n"; - print "
"; - print i18n::translate('Cleanup Places'); - print help_link('cleanup_places'); - print ""; - print "
".i18n::translate('Example of invalid place from your GEDCOM:')."
".PrintReady(nl2br($placesample[0])); - print "
\n"; - print "
"; - print "".i18n::translate('Detected invalid date formats, on cleanup these will be changed to format of DD MMM YYYY (eg. 1 JAN 2004).')."\n"; - print "\n"; - print ""; - print "
"; - print i18n::translate('Date format'); - print help_link('detected_date'); - - print ""; - if (isset ($datesample["choose"])) { - print ""; - } else - print ""; - print "
".i18n::translate('Example of invalid date from your GEDCOM:')."
".$datesample[0]; - print "
\n"; - print "
"; - print "".i18n::translate('ANSI file encoding detected. webtrees works best with files encoded in UTF-8.')."\n"; - print "\n"; - print ""; - print "
"; - print i18n::translate('Convert this ANSI encoded GEDCOM to UTF-8?'); - print help_link('detected_ansi2utf'); - print ""; - print "
\n"; - } - } - } else if (!$cleanup_needed) { - print i18n::translate('Valid GEDCOM detected. No cleanup required.'); - $import = true; - } else $import = true; - ?> - - - - - - - - -
-
-
"; - print "\"\""; - print " ".i18n::translate('Import Options').""; - print help_link('import_options'); - print "
"; - print "
"; - print "\n"; - - // NOTE: Time limit for import - print ""; - - // NOTE: Auto-click "Continue" button - print ""; - - // NOTE: change XREF to RIN, REFN, or Don't change - print "\n"; - - print ""; - print ""; - print ""; - print ""; - print ""; - print ""; - print ""; - print ""; - print "
"; - print i18n::translate('Time limit:'); - print help_link('time_limit'); - print "\n"; - print "
"; - print i18n::translate('Automatically press «Continue» button'); - print help_link('autoContinue'); - print ""; - print "
"; - print i18n::translate('Change Individual ID to:'); - print help_link('change_indi2id'); - print ""; - if ($startimport == "true") { - if ($xreftype == "NA") - print i18n::translate_c('Do not change Individual ID', 'Do not change'); - else - print $xreftype; - } else { - print ""; - } - print "
"; - print "
"; - print "
\n"; - print "".i18n::translate('Import Progress...').""; - print "
\n"; - print "
1%
\n"; - print "
\n"; - print "
\n"; - print "
 
"; - print "
 
 
"; - print "
"; - print "\n
"; - print "
\n"; - if ($timelimit == 0) - print "".i18n::translate('Time limit:')." ".i18n::translate('None').""; - else - print "".i18n::translate('Time limit:')." ".$timelimit." ".i18n::translate('sec.').""; - print "
\n"; - print "
1%
\n"; - print "
\n"; - print "
\n"; - print "
 
"; - print "
 
 
"; - print "
"; - flush(); - } -//-- end of setup_progress_bar function - -if (!isset ($stage)) -$stage = 0; -if ((empty ($ged)) || (!get_id_from_gedcom($ged))) { - $ged = $GEDCOM; -} -$ged_id=get_id_from_gedcom($ged); -$temp = $THEME_DIR; -$GEDCOM_FILE = get_gedcom_setting($ged_id, 'path'); -$FILE = $ged; -$TITLE = get_gedcom_setting($ged_id, 'title'); -require get_config_file($ged_id); - -$temp2 = $THEME_DIR; -$THEME_DIR = $temp; -$THEME_DIR = $temp2; - -if (isset ($GEDCOM_FILE)) { - if ((!isFileExternal($GEDCOM_FILE)) && (!file_exists($GEDCOM_FILE))) { - print "Could not locate gedcom file at $GEDCOM_FILE
\n"; - unset ($GEDCOM_FILE); - } -} - -if ($stage == 0) { - $_SESSION["resumed"] = 0; - if (file_exists($INDEX_DIRECTORY.basename($GEDCOM_FILE).".new")) - unlink($INDEX_DIRECTORY.basename($GEDCOM_FILE).".new"); - empty_database($ged_id, $keepmedia); - $stage = 1; -} -flush(); - -// Importing generates 1000's of separate updates, and in auto-commit mode, each of these -// is a separate transaction, which must be flushed to disk. This limits writes to approx -// 100 per second (on a typical 6000 RPM disk). -// By wrapping it all in one transaction, we only have one disk flush. -WT_DB::exec("START TRANSACTION"); - -if ($stage == 1) { - @ set_time_limit($timelimit); - //-- make sure that we are working with the true time limit - //-- commented out for now because PHP does not seem to be reporting it correctly on Linux - //$timelimit = ini_get("max_execution_time"); - - $FILE_SIZE = filesize($GEDCOM_FILE); - print ""; - print i18n::translate('Reading GEDCOM file')." ".$GEDCOM_FILE; - print ""; - print ""; - - print i18n::translate('The status bars below will let you know how the Import is progressing. If the time limit runs out the Import will be stopped and you will be asked to press a Continue button. If you don\'t see the Continue button, you must restart the Import with a smaller time limit value.'); - //print ""; - setup_progress_bar($FILE_SIZE); - //print ""; - flush(); - - // ------------------------------------------------------ Begin importing data - $i = 0; - - //-- as we are importing the file, a new file is being written to store any - //-- changes that might have occurred to the gedcom file (eg. conversion of - //-- media objects). After the import is complete the new file is - //-- copied over the old file. - //-- The records are written during the import_record() method and the - //-- update_media() method - //-- open handle to read file - $fpged = fopen($GEDCOM_FILE, "rb"); - //-- open handle to write changed file - $fpnewged = fopen($INDEX_DIRECTORY.basename($GEDCOM_FILE).".new", "ab"); - $BLOCK_SIZE = 1024 * 4; //-- 4k bytes per read (4kb is usually the page size of a virtual memory system) - //-- resume a halted import from the session - if (!empty ($_SESSION["resumed"])) { - $import_stats=$_SESSION['import_stats']; - $start_time =$_SESSION['start_time']; - $TOTAL_BYTES =$_SESSION["TOTAL_BYTES"]; - $fcontents = $_SESSION["fcontents"]; - $media_count = $_SESSION["media_count"]; - $found_ids = $_SESSION["found_ids"]; - $MAX_IDS = $_SESSION["MAX_IDS"]; - $autoContinue = $_SESSION["autoContinue"]; - fseek($fpged, $TOTAL_BYTES); - } else { - $fcontents = ""; - $TOTAL_BYTES = 0; - $media_count = 0; - $MAX_IDS = array(); - $_SESSION["resumed"] = 1; - $import_stats=array(); - $start_time=microtime(true); - } - while (!feof($fpged)) { - $temp = fread($fpged, $BLOCK_SIZE); - $fcontents .= $temp; - $TOTAL_BYTES += strlen($temp); - $pos1 = 0; - while ($pos1 !== false) { - //-- find the start of the next record - $pos2 = strpos($fcontents, "\n0", $pos1 +1); - while ((!$pos2) && (!feof($fpged))) { - $temp = fread($fpged, $BLOCK_SIZE); - $fcontents .= $temp; - $TOTAL_BYTES += strlen($temp); - $pos2 = strpos($fcontents, "\n0", $pos1 +1); - } - - //-- pull the next record out of the file - if ($pos2) { - $indirec = substr($fcontents, $pos1, $pos2 - $pos1); - } else { - $indirec = substr($fcontents, $pos1); - } - - try { - $record_type=import_record($indirec, $ged_id, false); - } catch (PDOException $ex) { - // Import errors are likely to be caused by duplicate records. - // There is no safe way of handling these. Just display them - // and let the user decide. - echo '
', $ex->getMessage(), '
'; - echo '
', WT_GEDCOM, ': ', $ged_id, '
'; - echo '
', htmlspecialchars($indirec), '
'; - // Don't let the error message disappear off the screen. - $autoContinue=false; - $record_type=i18n::translate('invalid'); - } - - // Generate import statistics - if (!isset($import_stats[$record_type])) { - $import_stats[$record_type]=array( - 'records'=>0, - 'bytes' =>0, - 'seconds'=>0 - ); - } - $end_time=microtime(true); - $import_stats[$record_type]['records']++; - $import_stats[$record_type]['bytes' ]+=$pos2 ? $pos2-$pos1 : strlen($indirec); - $import_stats[$record_type]['seconds']+=$end_time-$start_time; - $start_time=$end_time; - - //-- move the cursor to the start of the next record - $pos1 = $pos2; - - $i ++; - - //-- update the progress bars at every 10 records - if ($i % 10 == 0) { - $newtime = time(); - $exectime = $newtime - $oldtime; - echo WT_JS_START, "update_progress($TOTAL_BYTES, $exectime);", WT_JS_END; - flush(); - } else { - print ' '; - } - - //-- check if we are getting close to timing out - if ($i % 5 == 0) { - //-- keep the browser informed by sending more data - print "\n"; - $newtime = time(); - $exectime = $newtime - $oldtime; - if (($timelimit != 0) && ($timelimit - $exectime) < 2) { - $importtime = $importtime + $exectime; - $fcontents = substr($fcontents, $pos2); - //-- store the resume information in the session - $_SESSION['import_stats']=$import_stats; - $_SESSION['start_time']=$start_time; - $_SESSION["media_count"] = $media_count; - $_SESSION["TOTAL_BYTES"] = $TOTAL_BYTES; - $_SESSION["fcontents"] = $fcontents; - $_SESSION["importtime"] = $importtime; - $_SESSION["MAX_IDS"] = $MAX_IDS; - $_SESSION["found_ids"] = $found_ids; - $_SESSION["autoContinue"] = $autoContinue; - - //-- close the file connection - fclose($fpged); - fclose($fpnewged); - $_SESSION["resumed"]++; - print "\n"; - ?> - - - - - - -
- -
- - - Unable to create backup of the GEDCOM file at ".$INDEX_DIRECTORY.basename($GEDCOM_FILE).".bak
"; - //unlink($GEDCOM_FILE); - $res = @ copy($INDEX_DIRECTORY.basename($GEDCOM_FILE).".new", $GEDCOM_FILE); - if (!$res) { - print "Unable to copy updated GEDCOM file ".$INDEX_DIRECTORY.basename($GEDCOM_FILE).".new to ".$GEDCOM_FILE."
"; - } else { - @unlink($INDEX_DIRECTORY.basename($GEDCOM_FILE).".new"); - $logline = AddToLog($GEDCOM_FILE." updated", 'config'); - } - $newtime = time(); - $exectime = $newtime - $oldtime; - $importtime = $importtime + $exectime; - $exec_text = i18n::translate('Execution time:'); - $go_pedi = i18n::translate('Click here to go to the Pedigree tree.'); - $go_welc = i18n::translate('Home Page'); - if (WT_LOCALE == "fr" || WT_LOCALE == "it") { // TODO Just escape it properly! - echo WT_JS_START, "complete_progress($importtime, \"$exec_text\", \"$go_pedi\", \"$go_welc\");", WT_JS_END; - } else { - echo WT_JS_START, "complete_progress($importtime, '$exec_text', '$go_pedi', '$go_welc');", WT_JS_END; - } - flush(); - - // Import Statistics - $show_table1 = ""; - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= ""; - $total_seconds=0; - $total_bytes =0; - $total_records=0; - foreach ($import_stats as $type=>$stats) { - $total_seconds+=$stats['seconds']; - $total_bytes +=$stats['bytes']; - $total_records+=$stats['records']; - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= ""; - } - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= ""; - $show_table1 .= "
".i18n::translate('Import')."
".i18n::translate('Execution time:')."".i18n::translate('Bytes read:')."".i18n::translate('Found record')."".i18n::translate('Type')."
".sprintf("%.2f %s", $stats['seconds'], i18n::translate('sec.'))."".$stats['bytes']."".$stats['records']."".$type."
".sprintf("%.2f %s", $total_seconds, i18n::translate('sec.'))."".$total_bytes.WT_JS_START."update_progress($total_bytes, $exectime);".WT_JS_END; - $show_table1 .= "". $total_records." 
"; - echo "", i18n::translate('Import Statistics'), ""; - print ""; - print "
"; - if (isset ($skip_table)) { - print "
..."; - } else { - print $show_table1; - } - print "
"; - // NOTE: Finished Links - import_max_ids($ged_id, $MAX_IDS); - set_gedcom_setting($ged_id, 'imported', true); - print ""; - - $record_count = 0; - $_SESSION["resumed"] = 0; - unset ($_SESSION['import_stats']); - unset ($_SESSION["TOTAL_BYTES"]); - unset ($_SESSION["fcontents"]); -} -} -?> - -  "; - if ($verify && $startimport != "true") - print ""; - ?> - - -
- -- cgit v1.3