From 6504d09886aa62c1b32d336c766ed947187c30e2 Mon Sep 17 00:00:00 2001 From: jacob Date: Fri, 4 Jun 2010 20:23:59 +0000 Subject: [PATCH] Added updates for converting from Latin1 to utf8 character encoding --- admin/award_awards.php | 20 +++---- admin/communication.php | 28 +++------- admin/gettranslation.php | 6 +-- admin/project_editor.php | 6 +-- admin/reports.inc.php | 2 - admin/settranslation.php | 4 +- admin/student_editor.php | 38 ++++++------- common.inc.php | 9 ++-- db/db.code.version.txt | 2 +- db/db.update.173.php | 112 +++++++++++++++++++++++++++++++++++++++ db/db.update.173.sql | 0 db/db_update.php | 4 +- install2.php | 4 +- user_personal.php | 2 +- 14 files changed, 160 insertions(+), 77 deletions(-) create mode 100644 db/db.update.173.php create mode 100644 db/db.update.173.sql diff --git a/admin/award_awards.php b/admin/award_awards.php index 33dc1f0..55dfda0 100644 --- a/admin/award_awards.php +++ b/admin/award_awards.php @@ -32,11 +32,6 @@ $id = intval($_GET['id']); $q=mysql_query("SELECT * FROM award_awards WHERE id='$id'"); $ret = mysql_fetch_assoc($q); - //json_encode NEEDS UTF8 DATA, but we store it in the database as ISO :( - foreach($ret AS $k=>$v) { - $ret[$k]=iconv("ISO-8859-1","UTF-8",$v); - } - //echo iconv("ISO-8859-1","UTF-8",json_encode($ret)); echo json_encode($ret); exit; @@ -60,18 +55,18 @@ $q = "UPDATE award_awards SET award_types_id='".intval($_POST['award_types_id'])."', - presenter='".mysql_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['presenter'])))."', + presenter='".mysql_escape_string(stripslashes($_POST['presenter']))."', excludefromac='".(($_POST['excludefromac'] == 1) ? 1 : 0)."', cwsfaward='".(($_POST['cwsfaward'] == 1) ? 1 : 0)."', self_nominate='".(($_POST['self_nominate'] == 'yes') ? 'yes' : 'no')."', schedule_judges='".(($_POST['schedule_judges'] == 'yes') ? 'yes' : 'no')."', - description='".mysql_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['description'])))."' "; + description='".mysql_escape_string(stripslashes($_POST['description']))."' "; if(array_key_exists('name', $_POST)) { /* These values may be disabled, if they name key exists, assume * they aren't disabled and save them too */ - $q .= ",name='".mysql_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['name'])))."', - criteria='".mysql_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['criteria'])))."', + $q .= ",name='".mysql_escape_string(stripslashes($_POST['name']))."', + criteria='".mysql_escape_string(stripslashes($_POST['criteria']))."', sponsors_id='".intval($_POST['sponsors_id'])."' "; } $q .= "WHERE id='$id'"; @@ -160,9 +155,6 @@ $q = mysql_query("SELECT * FROM award_prizes WHERE award_awards_id='$id' ORDER BY `order`"); } while($r=mysql_fetch_assoc($q)) { - foreach($r AS $k=>$v) { - $r[$k]=iconv("ISO-8859-1","UTF-8",$v); - } $ret[] = $r; } echo json_encode($ret); @@ -172,7 +164,7 @@ $q = mysql_query("SELECT * FROM award_prizes WHERE id='$id'"); $ret=mysql_fetch_assoc($q); foreach($ret AS $k=>$v) { - $ret[$k]=iconv("ISO-8859-1","UTF-8",$v); + $ret[$k]=$v; } echo json_encode($ret); exit; @@ -192,7 +184,7 @@ case 'prize_save': $id = intval($_POST['id']); $q="UPDATE award_prizes SET - prize='".mysql_escape_string(stripslashes(iconv("UTF-8","ISO-8859-1",$_POST['prize'])))."', + prize='".mysql_escape_string(stripslashes("UTF-8","ISO-8859-1",$_POST['prize']))."', cash='".intval($_POST['cash'])."', scholarship='".intval($_POST['scholarship'])."', value='".intval($_POST['value'])."', diff --git a/admin/communication.php b/admin/communication.php index c052b5c..8fc1c25 100644 --- a/admin/communication.php +++ b/admin/communication.php @@ -136,28 +136,12 @@ case 'dialog_choose': case 'email_save': $id = intval($_POST['emails_id']); - //we need to character encode BEFORE we myql_real_escape_strintg - //otherwise, a smartquote ' will turn into a normal ' that ends up - //not being escaped! - $name=$_POST['name']; - $description=$_POST['description']; - $from=$_POST['from']; - $subject=$_POST['subject']; - $bodyhtml=$_POST['bodyhtml']; - - //add //TRANSLIT to approximate any characters (eg smartquotes) that it doesnt know - $bodyhtml=iconv("UTF-8","ISO-8859-1//TRANSLIT",$bodyhtml); - $name=iconv("UTF-8","ISO-8859-1//TRANSLIT",$name); - $description=iconv("UTF-8","ISO-8859-1//TRANSLIT",$description); - $from=iconv("UTF-8","ISO-8859-1//TRANSLIT",$from); - $subject=iconv("UTF-8","ISO-8859-1//TRANSLIT",$subject); - - //Now its safe to escape it for the db query - $name = mysql_real_escape_string(stripslashes($name)); - $description = mysql_real_escape_string(stripslashes($description)); - $from = mysql_real_escape_string(stripslashes($from)); - $subject = mysql_real_escape_string(stripslashes($subject)); - $bodyhtml = mysql_real_escape_string(stripslashes($bodyhtml)); + // escape all strings for the db query + $name = mysql_real_escape_string($_POST['name']); + $description = mysql_real_escape_string($_POST['description']); + $from = mysql_real_escape_string($_POST['from']); + $subject = mysql_real_escape_string($_POST['subject']); + $bodyhtml = mysql_real_escape_string($_POST['bodyhtml']); $type = mysql_real_escape_string($_POST['type']); $key = mysql_real_escape_string($_POST['key']); diff --git a/admin/gettranslation.php b/admin/gettranslation.php index 954c962..063e2af 100644 --- a/admin/gettranslation.php +++ b/admin/gettranslation.php @@ -28,11 +28,11 @@ user_auth_required('committee', 'admin'); $ret=array(); foreach($config['languages'] AS $l=>$ln) { if($l==$config['default_language']) continue; - $q=mysql_query("SELECT * FROM translations WHERE lang='$l' AND strmd5='".md5(iconv("ISO-8859-1","UTF-8",$_GET['str']))."'"); + $q=mysql_query("SELECT * FROM translations WHERE lang='$l' AND strmd5='".md5($_GET['str'])."'"); if($r=mysql_fetch_object($q)) - $ret[$l]=iconv("ISO-8859-1","UTF-8",$r->val); + $ret[$l]=$r->val; else - $ret[$l]=""; + $ret[$l]=""; } echo json_encode($ret); ?> diff --git a/admin/project_editor.php b/admin/project_editor.php index 78f62ef..4a0cb8a 100644 --- a/admin/project_editor.php +++ b/admin/project_editor.php @@ -113,13 +113,13 @@ function project_save() $title=stripslashes($_POST['title']); mysql_query("UPDATE projects SET ". - "title='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",$title))."', ". + "title='".mysql_escape_string($title)."', ". "projectdivisions_id='".intval($_POST['projectdivisions_id'])."', ". "language='".mysql_escape_string(stripslashes($_POST['language']))."', ". "req_table='".mysql_escape_string(stripslashes($_POST['req_table']))."', ". "req_electricity='".mysql_escape_string(stripslashes($_POST['req_electricity']))."', ". - "req_special='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['req_special'])))."', ". - "summary='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['summary'])))."', ". + "req_special='".mysql_escape_string(stripslashes($_POST['req_special']))."', ". + "summary='".mysql_escape_string(stripslashes($_POST['summary']))."', ". "summarycountok='$summarycountok',". "projectsort='".mysql_escape_string(stripslashes($_POST['projectsort']))."'". "WHERE id='".intval($_POST['id'])."'"); diff --git a/admin/reports.inc.php b/admin/reports.inc.php index 6aa4d38..3d87fdb 100644 --- a/admin/reports.inc.php +++ b/admin/reports.inc.php @@ -890,8 +890,6 @@ foreach($report_stock as $n=>$v) { } else { if($f == 'static_text') $v = $d['value']; - $v = iconv("ISO-8859-1//TRANSLIT", "UTF-8", $v); - $rep->label_text($d['x'], $d['y'], $d['w'], $d['h'], $v, $show_box, $d['align'], $d['valign'], $d['fontname'],$d['fontstyle'],$d['fontsize'], diff --git a/admin/settranslation.php b/admin/settranslation.php index c3fe921..dcdf269 100644 --- a/admin/settranslation.php +++ b/admin/settranslation.php @@ -34,9 +34,9 @@ foreach($config['languages'] AS $l=>$ln) { if($_POST['translate_'.$l]) { $q=mysql_query("SELECT * FROM translations WHERE lang='$l' AND strmd5='$m'"); if(mysql_num_rows($q)) - mysql_query("UPDATE translations SET val='".mysql_real_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['translate_'.$l])))."' WHERE lang='$l' AND strmd5='$m'"); + mysql_query("UPDATE translations SET val='".mysql_real_escape_string(stripslashes($_POST['translate_'.$l]))."' WHERE lang='$l' AND strmd5='$m'"); else - mysql_query("INSERT INTO translations (lang,strmd5,str,val) VALUES ('$l','$m','".mysql_real_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['translate_str_hidden'])))."','".mysql_escape_string(iconv("UTF-8","ISO-8859-1",stripslashes($_POST['translate_'.$l])))."')"); + mysql_query("INSERT INTO translations (lang,strmd5,str,val) VALUES ('$l','$m','".mysql_real_escape_string(stripslashes($_POST['translate_str_hidden']))."','".mysql_escape_string(stripslashes($_POST['translate_'.$l]))."')"); } else { mysql_query("DELETE FROM translations WHERE lang='$l' AND strmd5='$m'"); diff --git a/admin/student_editor.php b/admin/student_editor.php index 05fa386..83bdea9 100644 --- a/admin/student_editor.php +++ b/admin/student_editor.php @@ -116,13 +116,13 @@ function students_save() $dob=$_POST['year'][$x]."-".$_POST['month'][$x]."-".$_POST['day'][$x]; mysql_query("INSERT INTO students (registrations_id,firstname,lastname,sex,email,address,city,province,postalcode,phone,dateofbirth,grade,schools_id,tshirt,medicalalert,foodreq,teachername,teacheremail,year) VALUES (". "'".$registrations_id."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['firstname'][$x])))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['lastname'][$x])))."', ". + "'".mysql_escape_string(stripslashes($_POST['firstname'][$x]))."', ". + "'".mysql_escape_string(stripslashes($_POST['lastname'][$x]))."', ". "'".mysql_escape_string(stripslashes($_POST['sex'][$x]))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['email'][$x])))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['address'][$x])))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['city'][$x])))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['province'][$x])))."', ". + "'".mysql_escape_string(stripslashes($_POST['email'][$x]))."', ". + "'".mysql_escape_string(stripslashes($_POST['address'][$x]))."', ". + "'".mysql_escape_string(stripslashes($_POST['city'][$x]))."', ". + "'".mysql_escape_string(stripslashes($_POST['province'][$x]))."', ". "'".mysql_escape_string(stripslashes($_POST['postalcode'][$x]))."', ". "'".mysql_escape_string(stripslashes($_POST['phone'][$x]))."', ". "'$dob', ". @@ -131,8 +131,8 @@ function students_save() "'".mysql_escape_string(stripslashes($_POST['tshirt'][$x]))."', ". "'".mysql_escape_string(stripslashes($_POST['medicalalert'][$x]))."', ". "'".mysql_escape_string(stripslashes($_POST['foodreq'][$x]))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['teachername'][$x])))."', ". - "'".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['teacheremail'][$x])))."', ". + "'".mysql_escape_string(stripslashes($_POST['teachername'][$x]))."', ". + "'".mysql_escape_string(stripslashes($_POST['teacheremail'][$x]))."', ". "'".$config['FAIRYEAR']."')"); happy_("%1 %2 successfully added",array($_POST['firstname'][$x],$_POST['lastname'][$x])); @@ -151,25 +151,25 @@ function students_save() //UPDATE existing record $dob=$_POST['year'][$x]."-".$_POST['month'][$x]."-".$_POST['day'][$x]; mysql_query("UPDATE students SET ". - "firstname='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['firstname'][$x])))."', ". - "lastname='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['lastname'][$x])))."', ". + "firstname='".mysql_escape_string(stripslashes($_POST['firstname'][$x]))."', ". + "lastname='".mysql_escape_string(stripslashes($_POST['lastname'][$x]))."', ". "sex='".mysql_escape_string(stripslashes($_POST['sex'][$x]))."', ". - "email='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['email'][$x])))."', ". - "address='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['address'][$x])))."', ". - "city='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['city'][$x])))."', ". - "province='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['province'][$x])))."', ". + "email='".mysql_escape_string(stripslashes($_POST['email'][$x]))."', ". + "address='".mysql_escape_string(stripslashes($_POST['address'][$x]))."', ". + "city='".mysql_escape_string(stripslashes($_POST['city'][$x]))."', ". + "province='".mysql_escape_string(stripslashes($_POST['province'][$x]))."', ". "postalcode='".mysql_escape_string(stripslashes($_POST['postalcode'][$x]))."', ". "phone='".mysql_escape_string(stripslashes($_POST['phone'][$x]))."', ". "dateofbirth='$dob', ". "grade='".mysql_escape_string(stripslashes($_POST['grade'][$x]))."', ". $schoolquery. - "medicalalert='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['medicalalert'][$x])))."', ". - "foodreq='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['foodreq'][$x])))."', ". - "teachername='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['teachername'][$x])))."', ". - "teacheremail='".mysql_escape_string(iconv("UTF-8","ISO-8859-1//TRANSLIT",stripslashes($_POST['teacheremail'][$x])))."', ". + "medicalalert='".mysql_escape_string(stripslashes($_POST['medicalalert'][$x]))."', ". + "foodreq='".mysql_escape_string(stripslashes($_POST['foodreq'][$x]))."', ". + "teachername='".mysql_escape_string(stripslashes($_POST['teachername'][$x]))."', ". + "teacheremail='".mysql_escape_string(stripslashes($_POST['teacheremail'][$x]))."', ". "tshirt='".mysql_escape_string(stripslashes($_POST['tshirt'][$x]))."' ". "WHERE id='".$_POST['id'][$x]."'"); - happy_("%1 %2 successfully updated",array(iconv("UTF-8","ISO-8859-1//TRANSLIT",$_POST['firstname'][$x]),iconv("UTF-8","ISO-8859-1//TRANSLIT",$_POST['lastname'][$x]))); + happy_("%1 %2 successfully updated",array($_POST['firstname'][$x],$_POST['lastname'][$x])); } $x++; } diff --git a/common.inc.php b/common.inc.php index 9d710ef..51849f7 100644 --- a/common.inc.php +++ b/common.inc.php @@ -26,7 +26,7 @@ //which in many cases (like ysf-fsj.ca/sfiab) is UTF-8. This was causing a lot of the newly AJAX'd editors to fail on french characters, //becuase they were being encoded improperly. Ideally, all the databases will be switched to UTF-8, but thats not a near-term possibility, //so this is kind of a band-aid solution until we can make everything UTF8. Hope it doesnt break anything anywhere else! -header("Content-Type: text/html; charset=iso-8859-1"); +header("Content-Type: text/html; charset=UTF-8"); //set error reporting to not show notices, for some reason some people's installation dont set this by default //so we will set it in the code instead just to make sure @@ -100,9 +100,6 @@ if(!mysql_select_db($DBNAME)) exit; } -//this will silently fail on mysql 4.x, but is needed on mysql5.x to ensure we're only using iso-8859-1 (/latin1) encodings -@mysql_query("SET NAMES latin1"); - //find out the fair year and any other 'year=0' configuration parameters (things that dont change as the years go on) $q=@mysql_query("SELECT * FROM config WHERE year='0'"); @@ -421,7 +418,7 @@ function send_header($title="", $nav=null, $icon=null, $titletranslated=false) if($HEADER_SENT) return; else $HEADER_SENT=true; - echo "\n"; + echo "\n"; ?> @@ -728,7 +725,7 @@ function send_popup_header($title="") if($HEADER_SENT) return; else $HEADER_SENT=true; - echo "\n"; + echo "\n"; ?> diff --git a/db/db.code.version.txt b/db/db.code.version.txt index 730a054..c4597e5 100644 --- a/db/db.code.version.txt +++ b/db/db.code.version.txt @@ -1 +1 @@ -172 +173 diff --git a/db/db.update.173.php b/db/db.update.173.php new file mode 100644 index 0000000..eecb96d --- /dev/null +++ b/db/db.update.173.php @@ -0,0 +1,112 @@ + Array + ( + [0] => fieldname1 + [1] => fieldname2 + ... + ) + + [table2] => Array + ( + [0] => fieldname3 + [1] => fieldname4 + [2] => fieldname5 + ... + ) + ... + + Now we need to run through those tables one at a time and convert them + */ + $errorTally = 0; + echo "Updating records:\n"; + foreach($fields as $tableName => $fieldSet){ + // build the query that gives us the field values we need to update in this table, as well as the primary keys + $query = "SELECT `" . implode('`, `', $fieldSet) . '`'; + for($n = 0; $n < count($keys[$tableName]); $n++){ + $query .= ", `" . $keys[$tableName][$n] . "` AS __KEYFIELD_" . ($n + 1) . "__"; + } + $query .= " FROM $tableName"; + + // fetch all of those values + $updates = array(); + $data = mysql_query($query); + while($record = mysql_fetch_array($data)){ + $updates[] = $record; + } + + // now re-insert those values into the table + foreach($updates as $update){ + $query = "UPDATE $tableName SET"; + $useComma = false; + foreach($fieldSet as $fieldName){ + $fieldValue = $update[$fieldName]; + if($useComma){ + $query .= ","; + }else{ + $useComma = true; + } + $newValue = mb_convert_encoding($fieldValue, "UTF-8", "iso-8859-1"); + $query .= sprintf(" `%s` = '%s'", $fieldName, mysql_real_escape_string($newValue)); + } + $query .= " WHERE "; + for($n = 0; $n < count($keys[$tableName]); $n++){ + if($n > 0) $query .= " AND "; + $query .= "`" . $keys[$tableName][$n] . "` = '" . mysql_real_escape_string($update["__KEYFIELD_" . ($n + 1) . "__"]) . "'"; + } + $success = mysql_query($query); + if($success) echo '.'; + else{ + echo "\nFailed to execute query: $query\n"; + $errorTally ++; + } + } + unset($updates); + } + echo "\nComplete with $errorTally failed queries.\n"; + + // now drop the id column that we added to the committees_link table + $query = "ALTER TABLE `committees_link` DROP `id`"; + mysql_query($query); +} diff --git a/db/db.update.173.sql b/db/db.update.173.sql new file mode 100644 index 0000000..e69de29 diff --git a/db/db_update.php b/db/db_update.php index 14b5fce..1595441 100644 --- a/db/db_update.php +++ b/db/db_update.php @@ -22,7 +22,7 @@ else mysql_connect($DBHOST,$DBUSER,$DBPASS); mysql_select_db($DBNAME); -@mysql_query("SET NAMES latin1"); +@mysql_query("SET NAMES utf8"); $q=mysql_query("SELECT val FROM config WHERE var='DBVERSION' AND year='0'"); $r=mysql_fetch_object($q); $dbdbversion=$r->val; @@ -82,7 +82,7 @@ if($dbcodeversion && $dbdbversion) echo "db.update.$ver.sql detected - running...\n"; readfile("db.update.$ver.sql"); echo "\n"; - system("mysql --default-character-set=latin1 -h$DBHOST -u$DBUSER -p$DBPASS $DBNAME Done! installed database version $dbcodeversion
\n"; @@ -117,7 +117,7 @@ mysql_select_db($DBNAME); echo "db/db.full.$x.sql found
"; echo "Setting up database tables... "; - system("mysql --default-character-set=latin1 -h$DBHOST -u$DBUSER -p$DBPASS $DBNAME Done! installed database version $x
\n"; diff --git a/user_personal.php b/user_personal.php index 8019918..7a5f678 100644 --- a/user_personal.php +++ b/user_personal.php @@ -113,7 +113,7 @@ case 'save': $save = true; /* Set values */ foreach($fields as $f) { - $u[$f] = iconv("UTF-8","ISO-8859-1",stripslashes($_POST[$f])); + $u[$f] = stripslashes($_POST[$f]); /* Allow the user to clear a field regardless of regex */ if($u[$f] == '') continue;