From a25f0a04682c4c39c1947220c90af4118c713952 Mon Sep 17 00:00:00 2001 From: Greg Roach Date: Sun, 1 Feb 2015 00:01:36 +0000 Subject: Refactor classes to use namespaces, as per PSR-4. Replace GPL2 with GPL3. --- .../ezyang/htmlpurifier/docs/dev-advanced-api.html | 26 + .../ezyang/htmlpurifier/docs/dev-code-quality.txt | 29 + .../htmlpurifier/docs/dev-config-bcbreaks.txt | 79 ++ .../ezyang/htmlpurifier/docs/dev-config-naming.txt | 164 +++ .../htmlpurifier/docs/dev-config-schema.html | 412 +++++++ vendor/ezyang/htmlpurifier/docs/dev-flush.html | 68 ++ vendor/ezyang/htmlpurifier/docs/dev-includes.txt | 281 +++++ vendor/ezyang/htmlpurifier/docs/dev-naming.html | 83 ++ .../ezyang/htmlpurifier/docs/dev-optimization.html | 33 + vendor/ezyang/htmlpurifier/docs/dev-progress.html | 309 +++++ .../htmlpurifier/docs/dtd/xhtml1-transitional.dtd | 1201 ++++++++++++++++++++ .../htmlpurifier/docs/enduser-customize.html | 850 ++++++++++++++ vendor/ezyang/htmlpurifier/docs/enduser-id.html | 148 +++ .../ezyang/htmlpurifier/docs/enduser-overview.txt | 59 + .../ezyang/htmlpurifier/docs/enduser-security.txt | 18 + vendor/ezyang/htmlpurifier/docs/enduser-slow.html | 120 ++ vendor/ezyang/htmlpurifier/docs/enduser-tidy.html | 231 ++++ .../htmlpurifier/docs/enduser-uri-filter.html | 204 ++++ vendor/ezyang/htmlpurifier/docs/enduser-utf8.html | 1060 +++++++++++++++++ .../ezyang/htmlpurifier/docs/enduser-youtube.html | 153 +++ .../htmlpurifier/docs/entities/xhtml-lat1.ent | 196 ++++ .../htmlpurifier/docs/entities/xhtml-special.ent | 80 ++ .../htmlpurifier/docs/entities/xhtml-symbol.ent | 237 ++++ vendor/ezyang/htmlpurifier/docs/examples/basic.php | 23 + vendor/ezyang/htmlpurifier/docs/fixquotes.htc | 9 + vendor/ezyang/htmlpurifier/docs/index.html | 188 +++ .../ezyang/htmlpurifier/docs/proposal-colors.html | 49 + .../ezyang/htmlpurifier/docs/proposal-config.txt | 23 + .../htmlpurifier/docs/proposal-css-extraction.txt | 34 + .../ezyang/htmlpurifier/docs/proposal-errors.txt | 211 ++++ .../htmlpurifier/docs/proposal-filter-levels.txt | 137 +++ .../ezyang/htmlpurifier/docs/proposal-language.txt | 64 ++ .../htmlpurifier/docs/proposal-new-directives.txt | 44 + .../ezyang/htmlpurifier/docs/proposal-plists.txt | 218 ++++ .../htmlpurifier/docs/ref-content-models.txt | 50 + vendor/ezyang/htmlpurifier/docs/ref-css-length.txt | 30 + .../ezyang/htmlpurifier/docs/ref-devnetwork.html | 47 + .../htmlpurifier/docs/ref-html-modularization.txt | 166 +++ .../htmlpurifier/docs/ref-proprietary-tags.txt | 26 + vendor/ezyang/htmlpurifier/docs/ref-whatwg.txt | 26 + vendor/ezyang/htmlpurifier/docs/specimens/LICENSE | 10 + .../docs/specimens/html-align-to-css.html | 165 +++ vendor/ezyang/htmlpurifier/docs/specimens/img.png | Bin 0 -> 2138 bytes .../docs/specimens/jochem-blok-word.html | 129 +++ .../specimens/windows-live-mail-desktop-beta.html | 74 ++ vendor/ezyang/htmlpurifier/docs/style.css | 76 ++ 46 files changed, 7840 insertions(+) create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-advanced-api.html create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-code-quality.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-config-bcbreaks.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-config-naming.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-config-schema.html create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-flush.html create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-includes.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-naming.html create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-optimization.html create mode 100644 vendor/ezyang/htmlpurifier/docs/dev-progress.html create mode 100644 vendor/ezyang/htmlpurifier/docs/dtd/xhtml1-transitional.dtd create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-customize.html create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-id.html create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-overview.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-security.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-slow.html create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-tidy.html create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-uri-filter.html create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-utf8.html create mode 100644 vendor/ezyang/htmlpurifier/docs/enduser-youtube.html create mode 100644 vendor/ezyang/htmlpurifier/docs/entities/xhtml-lat1.ent create mode 100644 vendor/ezyang/htmlpurifier/docs/entities/xhtml-special.ent create mode 100644 vendor/ezyang/htmlpurifier/docs/entities/xhtml-symbol.ent create mode 100644 vendor/ezyang/htmlpurifier/docs/examples/basic.php create mode 100644 vendor/ezyang/htmlpurifier/docs/fixquotes.htc create mode 100644 vendor/ezyang/htmlpurifier/docs/index.html create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-colors.html create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-config.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-css-extraction.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-errors.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-filter-levels.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-language.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-new-directives.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/proposal-plists.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/ref-content-models.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/ref-css-length.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/ref-devnetwork.html create mode 100644 vendor/ezyang/htmlpurifier/docs/ref-html-modularization.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/ref-proprietary-tags.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/ref-whatwg.txt create mode 100644 vendor/ezyang/htmlpurifier/docs/specimens/LICENSE create mode 100644 vendor/ezyang/htmlpurifier/docs/specimens/html-align-to-css.html create mode 100644 vendor/ezyang/htmlpurifier/docs/specimens/img.png create mode 100644 vendor/ezyang/htmlpurifier/docs/specimens/jochem-blok-word.html create mode 100644 vendor/ezyang/htmlpurifier/docs/specimens/windows-live-mail-desktop-beta.html create mode 100644 vendor/ezyang/htmlpurifier/docs/style.css (limited to 'vendor/ezyang/htmlpurifier/docs') diff --git a/vendor/ezyang/htmlpurifier/docs/dev-advanced-api.html b/vendor/ezyang/htmlpurifier/docs/dev-advanced-api.html new file mode 100644 index 0000000000..5b7aaa3c8e --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-advanced-api.html @@ -0,0 +1,26 @@ + + + + + + + +Advanced API - HTML Purifier + + + +

Advanced API

+ +
Filed under Development
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

+ Please see Customize! +

+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/dev-code-quality.txt b/vendor/ezyang/htmlpurifier/docs/dev-code-quality.txt new file mode 100644 index 0000000000..bceedebc42 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-code-quality.txt @@ -0,0 +1,29 @@ + +Code Quality Issues + +Okay, face it. Programmers can get lazy, cut corners, or make mistakes. They +also can do quick prototypes, and then forget to rewrite them later. Well, +while I can't list mistakes in here, I can list prototype-like segments +of code that should be aggressively refactored. This does not list +optimization issues, that needs to be done after intense profiling. + +docs/examples/demo.php - ad hoc HTML/PHP soup to the extreme + +AttrDef - a lot of duplication, more generic classes need to be created; +a lot of strtolower() calls, no legit casing + Class - doesn't support Unicode characters (fringe); uses regular expressions + Lang - code duplication; premature optimization + Length - easily mistaken for CSSLength + URI - multiple regular expressions; missing validation for parts (?) + CSS - parser doesn't accept advanced CSS (fringe) + Number - constructor interface inconsistent with Integer +Strategy + FixNesting - cannot bubble nodes out of structures, duplicated checks + for special-case parent node + RemoveForeignElements - should be run in parallel with MakeWellFormed +URIScheme - needs to have callable generic checks + mailto - doesn't validate emails, doesn't validate querystring + news - doesn't validate opaque path + nntp - doesn't constrain path + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/dev-config-bcbreaks.txt b/vendor/ezyang/htmlpurifier/docs/dev-config-bcbreaks.txt new file mode 100644 index 0000000000..29a58ca2fc --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-config-bcbreaks.txt @@ -0,0 +1,79 @@ + +Configuration Backwards-Compatibility Breaks + +In version 4.0.0, the configuration subsystem (composed of the outwards +facing Config class, as well as the ConfigSchema and ConfigSchema_Interchange +subsystems), was significantly revamped to make use of property lists. +While most of the changes are internal, some internal APIs were changed for the +sake of clarity. HTMLPurifier_Config was kept completely backwards compatible, +although some of the functions were retrofitted with an unambiguous alternate +syntax. Both of these changes are discussed in this document. + + + +1. Outwards Facing Changes +-------------------------------------------------------------------------------- + +The HTMLPurifier_Config class now takes an alternate syntax. The general rule +is: + + If you passed $namespace, $directive, pass "$namespace.$directive" + instead. + +An example: + + $config->set('HTML', 'Allowed', 'p'); + +becomes: + + $config->set('HTML.Allowed', 'p'); + +New configuration options may have more than one namespace, they might +look something like %Filter.YouTube.Blacklist. While you could technically +set it with ('HTML', 'YouTube.Blacklist'), the logical extension +('HTML', 'YouTube', 'Blacklist') does not work. + +The old API will still work, but will emit E_USER_NOTICEs. + + + +2. Internal API Changes +-------------------------------------------------------------------------------- + +Some overarching notes: we've completely eliminated the notion of namespace; +it's now an informal construct for organizing related configuration directives. + +Also, the validation routines for keys (formerly "$namespace.$directive") +have been completely relaxed. I don't think it really should be necessary. + +2.1 HTMLPurifier_ConfigSchema + +First off, if you're interfacing with this class, you really shouldn't. +HTMLPurifier_ConfigSchema_Builder_ConfigSchema is really the only class that +should ever be creating HTMLPurifier_ConfigSchema, and HTMLPurifier_Config the +only class that should be reading it. + +All namespace related methods were removed; they are completely unnecessary +now. Any $namespace, $name arguments must be replaced with $key (where +$key == "$namespace.$name"), including for addAlias(). + +The $info and $defaults member variables are no longer indexed as +[$namespace][$name]; they are now indexed as ["$namespace.$name"]. + +All deprecated methods were finally removed, after having yelled at you as +an E_USER_NOTICE for a while now. + +2.2 HTMLPurifier_ConfigSchema_Interchange + +Member variable $namespaces was removed. + +2.3 HTMLPurifier_ConfigSchema_Interchange_Id + +Member variable $namespace and $directive removed; member variable $key added. +Any method that took $namespace, $directive now takes $key. + +2.4 HTMLPurifier_ConfigSchema_Interchange_Namespace + +Removed. + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/dev-config-naming.txt b/vendor/ezyang/htmlpurifier/docs/dev-config-naming.txt new file mode 100644 index 0000000000..66db5bce3c --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-config-naming.txt @@ -0,0 +1,164 @@ +Configuration naming + +HTML Purifier 4.0.0 features a new configuration naming system that +allows arbitrary nesting of namespaces. While there are certain cases +in which using two namespaces is obviously better (the canonical example +is where we were using AutoFormatParam to contain directives for AutoFormat +parameters), it is unclear whether or not a general migration to highly +namespaced directives is a good idea or not. + +== Case studies == + +=== Attr.* === + +We have a dead duck HTML.Attr.Name.UseCDATA which migrated before we decided +to think this out thoroughly. + +We currently have a large number of directives in the Attr.* namespace. +These directives tweak the behavior of some HTML attributes. They have +the properties: + +* While they apply to only one attribute at a time, the attribute can + span over multiple elements (not necessarily all attributes, either). + The information of which elements it impacts is either omitted or + informally stated (EnableID applies to all elements, DefaultImageAlt + applies to tags, AllowedRev doesn't say but only applies to a tags). + +* There is a certain degree of clustering that could be applied, especially + to the ID directives. The clustering could be done with respect to + what element/attribute was used, i.e. + + *.id -> EnableID, IDBlacklistRegexp, IDBlacklist, IDPrefixLocal, IDPrefix + img.src -> DefaultInvalidImage + img.alt -> DefaultImageAlt, DefaultInvalidImageAlt + bdo.dir -> DefaultTextDir + a.rel -> AllowedRel + a.rev -> AllowedRev + a.target -> AllowedFrameTargets + a.name -> Name.UseCDATA + +* The directives often reference generic attribute types that were specified + in the DTD/specification. However, some of the behavior specifically relies + on the fact that other use cases of the attribute are not, at current, + supported by HTML Purifier. + + AllowedRel, AllowedRev -> heavily specific; if ends up being + allowed, we will also have to give users specificity there (we also + want to preserve generality) DTD %Linktypes, HTML5 distinguishes + between and / + AllowedFrameTargets -> heavily specific, but also used by + and
. Transitional DTD %FrameTarget, not present in strict, + HTML5 calls them "browsing contexts" + Default*Image* -> as a default parameter, is almost entirely exlcusive + to + EnableID -> global attribute + Name.UseCDATA -> heavily specific, but has heavy other usage by + many things + +== AutoFormat.* == + +These have the fairly normal pluggable architecture that lends itself to +large amounts of namespaces (pluggability may be the key to figuring +out when gratuitous namespacing is good.) Properties: + +* Boolean directives are fair game for being namespaced: for example, + RemoveEmpty.RemoveNbsp triggers RemoveEmpty.RemoveNbsp.Exceptions, + the latter of which only makes sense when RemoveEmpty.RemoveNbsp + is set to true. (The same applies to RemoveNbsp too) + +The AutoFormat string is a bit long, but is the only bit of repeated +context. + +== Core.* == + +Core is the potpourri of directives, mostly regarding some minor behavioral +tweaks for HTML handling abilities. + + AggressivelyFixLt + ConvertDocumentToFragment + DirectLexLineNumberSyncInterval + LexerImpl + MaintainLineNumbers + Lexer + CollectErrors + Language + Error handling (Language is ostensibly a little more general, but + it's only used for error handling right now) + ColorKeywords + CSS and HTML + Encoding + EscapeNonASCIICharacters + Character encoding + EscapeInvalidChildren + EscapeInvalidTags + HiddenElements + RemoveInvalidImg + Lexing/Output + RemoveScriptContents + Deprecated + +== HTML.* == + + AllowedAttributes + AllowedElements + AllowedModules + Allowed + ForbiddenAttributes + ForbiddenElements + Element set tuning + BlockWrapper + Child def advanced twiddle + CoreModules + CustomDoctype + Advanced HTMLModuleManager twiddles + DefinitionID + DefinitionRev + Caching + Doctype + Parent + Strict + XHTML + Global environment + MaxImgLength + Attribute twiddle? (applies to two attributes) + Proprietary + SafeEmbed + SafeObject + Trusted + Extra functionality/tagsets + TidyAdd + TidyLevel + TidyRemove + Tidy + +== Output.* == + +These directly affect the output of Generator. These are all advanced +twiddles. + +== URI.* == + + AllowedSchemes + OverrideAllowedSchemes + Scheme tuning + Base + DefaultScheme + Host + Global environment + DefinitionID + DefinitionRev + Caching + DisableExternalResources + DisableExternal + DisableResources + Disable + Contextual/authority tuning + HostBlacklist + Authority tuning + MakeAbsolute + MungeResources + MungeSecretKey + Munge + Transformation behavior (munge can be grouped) + + diff --git a/vendor/ezyang/htmlpurifier/docs/dev-config-schema.html b/vendor/ezyang/htmlpurifier/docs/dev-config-schema.html new file mode 100644 index 0000000000..07aecd35ac --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-config-schema.html @@ -0,0 +1,412 @@ + + + + + + + + Config Schema - HTML Purifier + + + +

Config Schema

+ +
Filed under Development
+ +
HTML Purifier End-User Documentation
+ +

+ HTML Purifier has a fairly complex system for configuration. Users + interact with a HTMLPurifier_Config object to + set configuration directives. The values they set are validated according + to a configuration schema, HTMLPurifier_ConfigSchema. +

+ +

+ The schema is mostly transparent to end-users, but if you're doing development + work for HTML Purifier and need to define a new configuration directive, + you'll need to interact with it. We'll also talk about how to define + userspace configuration directives at the very end. +

+ +

Write a directive file

+ +

+ Directive files define configuration directives to be used by + HTML Purifier. They are placed in library/HTMLPurifier/ConfigSchema/schema/ + in the form Namespace.Directive.txt (I + couldn't think of a more descriptive file extension.) + Directive files are actually what we call StringHashes, + i.e. associative arrays represented in a string form reminiscent of + PHPT tests. Here's a + sample directive file, Test.Sample.txt: +

+ + 
Test.Sample
+TYPE: string/null
+DEFAULT: NULL
+ALLOWED: 'foo', 'bar'
+VALUE-ALIASES: 'baz' => 'bar'
+VERSION: 3.1.0
+--DESCRIPTION--
+This is a sample configuration directive for the purposes of the
+<code>dev-config-schema.html<code> documentation.
+--ALIASES--
+Test.Example
+ +

+ Each of these segments has a specific meaning: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyExampleDescription
IDTest.SampleThe name of the directive, in the form Namespace.Directive + (implicitly the first line)
TYPEstring/nullThe type of variable this directive accepts. See below for + details. You can also add /null to the end of + any basic type to allow null values too.
DEFAULTNULLA parseable PHP expression of the default value.
DESCRIPTIONThis is a...An HTML description of what this directive does.
VERSION3.1.0Recommended. The version of HTML Purifier this directive was added. + Directives that have been around since 1.0.0 don't have this, + but any new ones should.
ALIASESTest.ExampleOptional. A comma separated list of aliases for this directive. + This is most useful for backwards compatibility and should + not be used otherwise.
ALLOWED'foo', 'bar'Optional. Set of allowed value for a directive, + a comma separated list of parseable PHP expressions. This + is only allowed string, istring, text and itext TYPEs.
VALUE-ALIASES'baz' => 'bar'Optional. Mapping of one value to another, and + should be a comma separated list of keypair duples. This + is only allowed string, istring, text and itext TYPEs.
DEPRECATED-VERSION3.1.0Not shown. Indicates that the directive was + deprecated this version.
DEPRECATED-USETest.NewDirectiveNot shown. Indicates what new directive should be + used instead. Note that the directives will functionally be + different, although they should offer the same functionality. + If they are identical, use an alias instead.
EXTERNALCSSTidyNot shown. Indicates if there is an external library + the user will need to download and install to use this configuration + directive. As of right now, this is merely a Google-able name; future + versions may also provide links and instructions.
+ +

+ Some notes on format and style: +

+ + + +

+ Also, as promised, here is the set of possible types: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeExampleDescription
string'Foo'String without newlines
istring'foo'Case insensitive ASCII string without newlines
text"A\nb"String with newlines
itext"a\nb"Case insensitive ASCII string without newlines
int23Integer
float3.0Floating point number
booltrueBoolean
lookuparray('key' => true)Lookup array, used with isset($var[$key])
listarray('f', 'b')List array, with ordered numerical indexes
hasharray('key' => 'val')Associative array of keys to values
mixednew stdclassAny PHP variable is fine
+ +

+ The examples represent what will be returned out of the configuration + object; users have a little bit of leeway when setting configuration + values (for example, a lookup value can be specified as a list; + HTML Purifier will flip it as necessary.) These types are defined + in + library/HTMLPurifier/VarParser.php. +

+ +

+ For more information on what values are allowed, and how they are parsed, + consult + library/HTMLPurifier/ConfigSchema/InterchangeBuilder.php, as well + as + library/HTMLPurifier/ConfigSchema/Interchange/Directive.php for + the semantics of the parsed values. +

+ +

Refreshing the cache

+ +

+ You may have noticed that your directive file isn't doing anything + yet. That's because it hasn't been added to the runtime + HTMLPurifier_ConfigSchema instance. Run + maintenance/generate-schema-cache.php to fix this. + If there were no errors, you're good to go! Don't forget to add + some unit tests for your functionality! +

+ +

+ If you ever make changes to your configuration directives, you + will need to run this script again. +

+

Adding in-house schema definitions

+ +

+ Placing stuff directly in HTML Purifier's source tree is generally not a + good idea, so HTML Purifier 4.0.0+ has some facilities in place to make your + life easier. +

+ +

+ The first is to pass an extra parameter to maintenance/generate-schema-cache.php + with the location of your directory (relative or absolute path will do). For example, + if I'm storing my custom definitions in /var/htmlpurifier/myschema, run: + php maintenance/generate-schema-cache.php /var/htmlpurifier/myschema. +

+ +

+ Alternatively, you can create a small loader PHP file in the HTML Purifier base + directory named config-schema.php (this is the same directory + you would place a test-settings.php file). In this file, add + the following line for each directory you want to load: +

+ +
$builder->buildDir($interchange, '/var/htmlpurifier/myschema');
+ +

You can even load a single file using:

+ +
$builder->buildFile($interchange, '/var/htmlpurifier/myschema/MyApp.Directive.txt');
+ +

Storing custom definitions that you don't plan on sending back upstream in + a separate directory is definitely a good idea! Additionally, picking + a good namespace can go a long way to saving you grief if you want to use + someone else's change, but they picked the same name, or if HTML Purifier + decides to add support for a configuration directive that has the same name.

+ + + +

Errors

+ +

+ All directive files go through a rigorous validation process + through + library/HTMLPurifier/ConfigSchema/Validator.php, as well + as some basic checks during building. While + listing every error out here is out-of-scope for this document, we + can give some general tips for interpreting error messages. + There are two types of errors: builder errors and validation errors. +

+ +

Builder errors

+ +
+

+ Exception: Expected type string, got + integer in DEFAULT in directive hash 'Ns.Dir' +

+
+ +

+ You can identify a builder error by the keyword "directive hash." + These are the easiest to deal with, because they directly correspond + with your directive file. Find the offending directive file (which + is the directive hash plus the .txt extension), find the + offending index ("in DEFAULT" means the DEFAULT key) and fix the error. + This particular error would occur if your default value is not the same + type as TYPE. +

+ +

Validation errors

+ +
+

+ Exception: Alias 3 in valueAliases in directive + 'Ns.Dir' must be a string +

+
+ +

+ These are a little trickier, because we're not actually validating + your directive file, or even the direct string hash representation. + We're validating an Interchange object, and the error messages do + not mention any string hash keys. +

+ +

+ Nevertheless, it's not difficult to figure out what went wrong. + Read the "context" statements in reverse: +

+ +
+
in directive 'Ns.Dir'
+
This means we need to look at the directive file Ns.Dir.txt
+
in valueAliases
+
There's no key actually called this, but there's one that's close: + VALUE-ALIASES. Indeed, that's where to look.
+
Alias 3
+
The value alias that is equal to 3 is the culprit.
+
+ +

+ In this particular case, you're not allowed to alias integers values to + strings values. +

+ +

+ The most difficult part is translating the Interchange member variable (valueAliases) + into a directive file key (VALUE-ALIASES), but there's a one-to-one + correspondence currently. If the two formats diverge, any discrepancies + will be described in + library/HTMLPurifier/ConfigSchema/InterchangeBuilder.php. +

+ +

Internals

+ +

+ Much of the configuration schema framework's codebase deals with + shuffling data from one format to another, and doing validation on this + data. + The keystone of all of this is the HTMLPurifier_ConfigSchema_Interchange + class, which represents the purest, parsed representation of the schema. +

+ +

+ Hand-writing this data is unwieldy, however, so we write directive files. + These directive files are parsed by HTMLPurifier_StringHashParser + into HTMLPurifier_StringHashes, which then + are run through HTMLPurifier_ConfigSchema_InterchangeBuilder + to construct the interchange object. +

+ +

+ From the interchange object, the data can be siphoned into other forms + using HTMLPurifier_ConfigSchema_Builder subclasses. + For example, HTMLPurifier_ConfigSchema_Builder_ConfigSchema + generates a runtime HTMLPurifier_ConfigSchema object, + which HTMLPurifier_Config uses to validate its incoming + data. There is also an XML serializer, which is used to build documentation. +

+ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/dev-flush.html b/vendor/ezyang/htmlpurifier/docs/dev-flush.html new file mode 100644 index 0000000000..4a3a78351c --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-flush.html @@ -0,0 +1,68 @@ + + + + + + + + Flushing the Purifier - HTML Purifier + + + +

Flushing the Purifier

+ +
Filed under Development
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

+ If you've been poking around the various folders in HTML Purifier, + you may have noticed the maintenance directory. Almost + all of these scripts are devoted to flushing out the various caches + HTML Purifier uses. Normal users don't have to worry about this: + regular library usage is transparent. However, when doing development + work on HTML Purifier, you may find you have to flush one of the + caches. +

+ +

+ As a general rule of thumb, run flush.php whenever you make + any major changes, or when tests start mysteriously failing. + In more detail, run this script if: +

+ + + +

+ You can check out the corresponding scripts for more information on what they + do. +

+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/dev-includes.txt b/vendor/ezyang/htmlpurifier/docs/dev-includes.txt new file mode 100644 index 0000000000..d3382b5933 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-includes.txt @@ -0,0 +1,281 @@ + +INCLUDES, AUTOLOAD, BYTECODE CACHES and OPTIMIZATION + +The Problem +----------- + +HTML Purifier contains a number of extra components that are not used all +of the time, only if the user explicitly specifies that we should use +them. + +Some of these optional components are optionally included (Filter, +Language, Lexer, Printer), while others are included all the time +(Injector, URIFilter, HTMLModule, URIScheme). We will stipulate that these +are all developer specified: it is conceivable that certain Tokens are not +used, but this is user-dependent and should not be trusted. + +We should come up with a consistent way to handle these things and ensure +that we get the maximum performance when there is bytecode caches and +when there are not. Unfortunately, these two goals seem contrary to each +other. + +A peripheral issue is the performance of ConfigSchema, which has been +shown take a large, constant amount of initialization time, and is +intricately linked to the issue of includes due to its pervasive use +in our plugin architecture. + +Pros and Cons +------------- + +We will assume that user-based extensions will be included by them. + +Conditional includes: + Pros: + - User management is simplified; only a single directive needs to be set + - Only necessary code is included + Cons: + - Doesn't play nicely with opcode caches + - Adds complexity to standalone version + - Optional configuration directives are not exposed without a little + extra coaxing (not implemented yet) + +Include it all: + Pros: + - User management is still simple + - Plays nicely with opcode caches and standalone version + - All configuration directives are present + Cons: + - Lots of (how much?) extra code is included + - Classes that inherit from external libraries will cause compile + errors + +Build an include stub (Let's do this!): + Pros: + - Only necessary code is included + - Plays nicely with opcode caches and standalone version + - require (without once) can be used, see above + - Could further extend as a compilation to one file + Cons: + - Not implemented yet + - Requires user intervention and use of a command line script + - Standalone script must be chained to this + - More complex and compiled-language-like + - Requires a whole new class of system-wide configuration directives, + as configuration objects can be reused + - Determining what needs to be included can be complex (see above) + - No way of autodetecting dynamically instantiated classes + - Might be slow + +Include stubs +------------- + +This solution may be "just right" for users who are heavily oriented +towards performance. However, there are a number of picky implementation +details to work out beforehand. + +The number one concern is how to make the HTML Purifier files "work +out of the box", while still being able to easily get them into a form +that works with this setup. As the codebase stands right now, it would +be necessary to strip out all of the require_once calls. The only way +we could get rid of the require_once calls is to use __autoload or +use the stub for all cases (which might not be a bad idea). + + Aside + ----- + An important thing to remember, however, is that these require_once's + are valuable data about what classes a file needs. Unfortunately, there's + no distinction between whether or not the file is needed all the time, + or whether or not it is one of our "optional" files. Thus, it is + effectively useless. + + Deprecated + ---------- + One of the things I'd like to do is have the code search for any classes + that are explicitly mentioned in the code. If a class isn't mentioned, I + get to assume that it is "optional," i.e. included via introspection. + The choice is either to use PHP's tokenizer or use regexps; regexps would + be faster but a tokenizer would be more correct. If this ends up being + unfeasible, adding dependency comments isn't a bad idea. (This could + even be done automatically by search/replacing require_once, although + we'd have to manually inspect the results for the optional requires.) + + NOTE: This ends up not being necessary, as we're going to make the user + figure out all the extra classes they need, and only include the core + which is predetermined. + +Using the autoload framework with include stubs works nicely with +introspective classes: instead of having to have require_once inside +the function, we can let autoload do the work; we simply need to +new $class or accept the object straight from the caller. Handling filters +becomes a simple matter of ticking off configuration directives, and +if ConfigSchema spits out errors, adding the necessary includes. We could +also use the autoload framework as a fallback, in case the user forgets +to make the include, but doesn't really care about performance. + + Insight + ------- + All of this talk is merely a natural extension of what our current + standalone functionality does. However, instead of having our code + perform the includes, or attempting to inline everything that possibly + could be used, we boot the issue to the user, making them include + everything or setup the fallback autoload handler. + +Configuration Schema +-------------------- + +A common deficiency for all of the conditional include setups (including +the dynamically built include PHP stub) is that if one of this +conditionally included files includes a configuration directive, it +is not accessible to configdoc. A stopgap solution for this problem is +to have it piggy-back off of the data in the merge-library.php script +to figure out what extra files it needs to include, but if the file also +inherits classes that don't exist, we're in big trouble. + +I think it's high time we centralized the configuration documentation. +However, the type checking has been a great boon for the library, and +I'd like to keep that. The compromise is to use some other source, and +then parse it into the ConfigSchema internal format (sans all of those +nasty documentation strings which we really don't need at runtime) and +serialize that for future use. + +The next question is that of format. XML is very verbose, and the prospect +of setting defaults in it gives me willies. However, this may be necessary. +Splitting up the file into manageable chunks may alleviate this trouble, +and we may be even want to create our own format optimized for specifying +configuration. It might look like (based off the PHPT format, which is +nicely compact yet unambiguous and human-readable): + +Core.HiddenElements +TYPE: lookup +DEFAULT: array('script', 'style') // auto-converted during processing +--ALIASES-- +Core.InvisibleElements, Core.StupidElements +--DESCRIPTION-- +

+ Blah blah +

+ +The first line is the directive name, the lines after that prior to the +first --HEADER-- block are single-line values, and then after that +the multiline values are there. No value is restricted to a particular +format: DEFAULT could very well be multiline if that would be easier. +This would make it insanely easy, also, to add arbitrary extra parameters, +like: + +VERSION: 3.0.0 +ALLOWED: 'none', 'light', 'medium', 'heavy' // this is wrapped in array() +EXTERNAL: CSSTidy // this would be documented somewhere else with a URL + +The final loss would be that you wouldn't know what file the directive +was used in; with some clever regexps it should be possible to +figure out where $config->get($ns, $d); occurs. Reflective calls to +the configuration object is mitigated by the fact that getBatch is +used, so we can simply talk about that in the namespace definition page. +This might be slow, but it would only happen when we are creating +the documentation for consumption, and is sugar. + +We can put this in a schema/ directory, outside of HTML Purifier. The serialized +data gets treated like entities.ser. + +The final thing that needs to be handled is user defined configurations. +They can be added at runtime using ConfigSchema::registerDirectory() +which globs the directory and grabs all of the directives to be incorporated +in. Then, the result is saved. We may want to take advantage of the +DefinitionCache framework, although it is not altogether certain what +configuration directives would be used to generate our key (meta-directives!) + + Further thoughts + ---------------- + Our master configuration schema will only need to be updated once + every new version, so it's easily versionable. User specified + schema files are far more volatile, but it's far too expensive + to check the filemtimes of all the files, so a DefinitionRev style + mechanism works better. However, we can uniquely identify the + schema based on the directories they loaded, so there's no need + for a DefinitionId until we give them full programmatic control. + + These variables should be directly incorporated into ConfigSchema, + and ConfigSchema should handle serialization. Some refactoring will be + necessary for the DefinitionCache classes, as they are built with + Config in mind. If the user changes something, the cache file gets + rebuilt. If the version changes, the cache file gets rebuilt. Since + our unit tests flush the caches before we start, and the operation is + pretty fast, this will not negatively impact unit testing. + +One last thing: certain configuration directives require that files +get added. They may even be specified dynamically. It is not a good idea +for the HTMLPurifier_Config object to be used directly for such matters. +Instead, the userland code should explicitly perform the includes. We may +put in something like: + +REQUIRES: HTMLPurifier_Filter_ExtractStyleBlocks + +To indicate that if that class doesn't exist, and the user is attempting +to use the directive, we should fatally error out. The stub includes the core files, +and the user includes everything else. Any reflective things like new +$class would be required to tie in with the configuration. + +It would work very well with rarely used configuration options, but it +wouldn't be so good for "core" parts that can be disabled. In such cases +the core include file would need to be modified, and the only way +to properly do this is use the configuration object. Once again, our +ability to create cache keys saves the day again: we can create arbitrary +stub files for arbitrary configurations and include those. They could +even be the single file affairs. The only thing we'd need to include, +then, would be HTMLPurifier_Config! Then, the configuration object would +load the library. + + An aside... + ----------- + One questions, however, the wisdom of letting PHP files write other PHP + files. It seems like a recipe for disaster, or at least lots of headaches + in highly secured setups, where PHP does not have the ability to write + to its root. In such cases, we could use sticky bits or tell the user + to manually generate the file. + + The other troublesome bit is actually doing the calculations necessary. + For certain cases, it's simple (such as URIScheme), but for AttrDef + and HTMLModule the dependency trees are very complex in relation to + %HTML.Allowed and friends. I think that this idea should be shelved + and looked at a later, less insane date. + +An interesting dilemma presents itself when a configuration form is offered +to the user. Normally, the configuration object is not accessible without +editing PHP code; this facility changes thing. The sensible thing to do +is stipulate that all classes required by the directives you allow must +be included. + +Unit testing +------------ + +Setting up the parsing and translation into our existing format would not +be difficult to do. It might represent a good time for us to rethink our +tests for these facilities; as creative as they are, they are often hacky +and require public visibility for things that ought to be protected. +This is especially applicable for our DefinitionCache tests. + +Migration +--------- + +Because we are not *adding* anything essentially new, it should be trivial +to write a script to take our existing data and dump it into the new format. +Well, not trivial, but fairly easy to accomplish. Primary implementation +difficulties would probably involve formatting the file nicely. + +Backwards-compatibility +----------------------- + +I expect that the ConfigSchema methods should stick around for a little bit, +but display E_USER_NOTICE warnings that they are deprecated. This will +require documentation! + +New stuff +--------- + +VERSION: Version number directive was introduced +DEPRECATED-VERSION: If the directive was deprecated, when was it deprecated? +DEPRECATED-USE: If the directive was deprecated, what should the user use now? +REQUIRES: What classes does this configuration directive require, but are + not part of the HTML Purifier core? + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/dev-naming.html b/vendor/ezyang/htmlpurifier/docs/dev-naming.html new file mode 100644 index 0000000000..cea4b006f2 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-naming.html @@ -0,0 +1,83 @@ + + + + + + + +Naming Conventions - HTML Purifier + + + +

Naming Conventions

+ +
Filed under Development
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

The classes in this library follow a few naming conventions, which may +help you find the correct functionality more quickly. Here they are:

+ +
+ +
All classes occupy the HTMLPurifier pseudo-namespace.
+
This means that all classes are prefixed with HTMLPurifier_. As such, all + names under HTMLPurifier_ are reserved. I recommend that you use the name + HTMLPurifierX_YourName_ClassName, especially if you want to take advantage + of HTMLPurifier_ConfigDef.
+ +
All classes correspond to their path if library/ was in the include path
+
HTMLPurifier_AttrDef is located at HTMLPurifier/AttrDef.php; replace + underscores with slashes and append .php and you'll have the location of + the class.
+ +
Harness and Test are reserved class names for unit tests
+
The suffix Test indicates that the class is a subclass of UnitTestCase + (of the Simpletest library) and is testable. "Harness" indicates a subclass + of UnitTestCase that is not meant to be run but to be extended into + concrete test cases and contains custom test methods (i.e. assert*())
+ +
Class names do not necessarily represent inheritance hierarchies
+
While we try to reflect inheritance in naming to some extent, it is not + guaranteed (for instance, none of the classes inherit from HTMLPurifier, + the base class). However, all class files have the require_once + declarations to whichever classes they are tightly coupled to.
+ +
Strategy has a meaning different from the Gang of Four pattern
+
In Design Patterns, the Gang of Four describes a Strategy object as + encapsulating an algorithm so that they can be switched at run-time. While + our strategies are indeed algorithms, they are not meant to be substituted: + all must be present in order for proper functioning.
+ +
Abbreviations are avoided
+
We try to avoid abbreviations as much as possible, but in some cases, + abbreviated version is more readable than the full version. Here, we + list common abbreviations: +
    +
  • Attr to Attributes (note that it is plural, i.e. $attr = array())
    • +
  • Def to Definition
    • +
  • $ret is the value to be returned in a function
    • +
+
+ +
Ambiguity concerning the definition of Def/Definition
+
While a definition normally defines the structure/acceptable values of + an entity, most of the definitions in this application also attempt + to validate and fix the value. I am unsure of a better name, as + "Validator" would exclude fixing the value, "Fixer" doesn't invoke + the proper image of "fixing" something, and "ValidatorFixer" is too long! + Some other suggestions were "Handler", "Reference", "Check", "Fix", + "Repair" and "Heal".
+ +
Transform not Transformer
+
Transform is both a noun and a verb, and thus we define a "Transform" as + something that "transforms," leaving "Transformer" (which sounds like an + electrical device/robot toy).
+ +
+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/dev-optimization.html b/vendor/ezyang/htmlpurifier/docs/dev-optimization.html new file mode 100644 index 0000000000..78f5658133 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-optimization.html @@ -0,0 +1,33 @@ + + + + + + + +Optimization - HTML Purifier + + + +

Optimization

+ +
Filed under Development
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

Here are some possible optimization techniques we can apply to code sections if +they turn out to be slow. Be sure not to prematurely optimize: if you get +that itch, put it here!

+ + + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/dev-progress.html b/vendor/ezyang/htmlpurifier/docs/dev-progress.html new file mode 100644 index 0000000000..105896ed60 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dev-progress.html @@ -0,0 +1,309 @@ + + + + + + + +Implementation Progress - HTML Purifier + + + + + +

Implementation Progress

+ +
Filed under Development
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

+ Warning: This table is kept for historical purposes and + is not being actively updated. +

+ +

Key

+ + + + + + + + +
Implemented
Partially implemented
Not priority to implement
Dangerous attribute/property
Present in CSS1
Feature, requires extra work
+ +

CSS

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameNotes
Standard
background-colorCOMPOSITE(<color>, transparent)
backgroundSHORTHAND, currently alias for background-color
borderSHORTHAND, MULTIPLE
border-colorMULTIPLE
border-styleMULTIPLE
border-widthMULTIPLE
border-*SHORTHAND
border-*-colorCOMPOSITE(<color>, transparent)
border-*-styleENUM(none, hidden, dotted, dashed, + solid, double, groove, ridge, inset, outset)
border-*-widthCOMPOSITE(<length>, thin, medium, thick)
clearENUM(none, left, right, both)
color<color>
floatENUM(left, right, none), May require layout + precautions with clear
fontSHORTHAND
font-familyCSS validator may complain if fallback font + family not specified
font-sizeCOMPOSITE(<absolute-size>, + <relative-size>, <length>, <percentage>)
font-styleENUM(normal, italic, oblique)
font-variantENUM(normal, small-caps)
font-weightENUM(normal, bold, bolder, lighter, + 100, 200, 300, 400, 500, 600, 700, 800, 900), maybe special code for + in-between integers
letter-spacingCOMPOSITE(<length>, normal)
line-heightCOMPOSITE(<number>, + <length>, <percentage>, normal)
list-style-positionENUM(inside, outside), + Strange behavior in browsers
list-style-typeENUM(...), + Well-supported values are: disc, circle, square, + decimal, lower-roman, upper-roman, lower-alpha and upper-alpha. See also + CSS 3. Mostly IE lack of support.
list-styleSHORTHAND
marginMULTIPLE
margin-*COMPOSITE(<length>, + <percentage>, auto)
paddingMULTIPLE
padding-*COMPOSITE(<length>(positive), + <percentage>(positive))
text-alignENUM(left, right, + center, justify)
text-decorationNo blink (argh my eyes), not + enum, can be combined (composite sorta): underline, overline, + line-through
text-indentCOMPOSITE(<length>, + <percentage>)
text-transformENUM(capitalize, uppercase, + lowercase, none)
widthCOMPOSITE(<length>, + <percentage>, auto), Interesting
word-spacingCOMPOSITE(<length>, auto), + IE 5 no support
Table
border-collapseENUM(collapse, seperate)
border-spaceMULTIPLE
caption-sideENUM(top, bottom)
empty-cellsENUM(show, hide), No IE support makes this useless, + possible fix with &nbsp;? Unknown release milestone.
table-layoutENUM(auto, fixed)
vertical-alignCOMPOSITE(ENUM(baseline, sub, + super, top, text-top, middle, bottom, text-bottom), <percentage>, + <length>) Also applies to others with explicit height
Absolute positioning, unknown release milestone
bottomDangerous, must be non-negative to even be considered, + but it's still possible to arbitrarily position by running over.
left
right
top
clip-
positionENUM(static, relative, absolute, fixed) + relative not absolute?
z-indexDangerous
Unknown
background-imageDangerous
background-attachmentENUM(scroll, fixed), + Depends on background-image
background-positionDepends on background-image
cursorDangerous but fluffy
displayENUM(...), Dangerous but interesting; + will not implement list-item, run-in (Opera only) or table (no IE); + inline-block has incomplete IE6 support and requires -moz-inline-box + for Mozilla. Unknown target milestone.
heightInteresting, why use it? Unknown target milestone.
list-style-imageDangerous?
max-heightNo IE 5/6
min-height
max-width
min-width
orphansNo IE support
widowsNo IE support
overflowENUM, IE 5/6 almost (remove visible if set). Unknown target milestone.
page-break-afterENUM(auto, always, avoid, left, right), + IE 5.5/6 and Opera. Unknown target milestone.
page-break-beforeENUM(auto, always, avoid, left, right), + Mostly supported. Unknown target milestone.
page-break-insideENUM(avoid, auto), Opera only. Unknown target milestone.
quotesMay be dropped from CSS2, fairly useless for inline context
visibilityENUM(visible, hidden, collapse), + Dangerous
white-spaceENUM(normal, pre, nowrap, pre-wrap, + pre-line), Spotty implementation: + pre (no IE 5/6), nowrap (no IE 5, supported), + pre-wrap (only Opera), pre-line (no support). Fixable? Unknown target milestone.
Aural
azimuth-
cue-
cue-after-
cue-before-
elevation-
pause-after-
pause-before-
pause-
pitch-range-
pitch-
play-during-
richness-
speak-headerTable related
speak-numeral-
speak-punctuation-
speak-
speech-rate-
stress-
voice-family-
volume-
Will not implement
contentNot applicable for inline styles
counter-incrementNeeds content, Opera only
counter-resetNeeds content, Opera only
directionNo support
outline-colorIE Mac and Opera on outside, +Mozilla on inside and needs -moz-outline, no IE support.
outline-style
outline-width
outline
unicode-bidiNo support
+ +

Interesting Attributes

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AttributeTagsNotes
CSS
styleAllParser is reasonably functional. Status here doesn't count individual properties.
Questionable
accesskeyAMay interfere with main interface
tabindexAMay interfere with main interface
targetAConfig enabled, only useful for frame layouts, disallowed in strict
Miscellaneous
datetimeDEL, INSNo visible effect, ISO format
relALargely user-defined: nofollow, tag (see microformats)
revALargely user-defined: vote-*
axisTD, THW3C only: No browser implementation
charCOL, COLGROUP, TBODY, TD, TFOOT, TH, THEAD, TRW3C only: No browser implementation
headersTD, THW3C only: No browser implementation
scopeTD, THW3C only: No browser implementation
URI
citeBLOCKQUOTE, QFor attribution
DEL, INSLink to explanation why it changed
hrefA-
longdescIMG-
srcIMGRequired
Transform
alignCAPTION'caption-side' for top/bottom, 'text-align' for left/right
IMGSee specimens/html-align-to-css.html
TABLE
HR
H1, H2, H3, H4, H5, H6, PEquivalent style 'text-align'
altIMGRequired, insert image filename if src is present or default invalid image text
bgcolorTABLESuperset style 'background-color'
TRSuperset style 'background-color'
TD, THSuperset style 'background-color'
borderIMGEquivalent style border:[number]px solid
clearBRNear-equiv style 'clear', transform 'all' into 'both'
compactDL, OL, ULBoolean, needs custom CSS class; rarely used anyway
dirBDORequired, insert ltr (or configuration value) if none
heightTD, THNear-equiv style 'height', needs px suffix if original was in pixels
hspaceIMGNear-equiv styles 'margin-top' and 'margin-bottom', needs px suffix
lang*Copy value to xml:lang
nameIMGTurn into ID
ATurn into ID
noshadeHRBoolean, style 'border-style:solid;'
nowrapTD, THBoolean, style 'white-space:nowrap;' (not compat with IE5)
sizeHRNear-equiv 'height', needs px suffix if original was pixels
srcIMGRequired, insert blank or default img if not set
startOLPoorly supported 'counter-reset', allowed in loose, dropped in strict
typeLIEquivalent style 'list-style-type', different allowed values though. (needs testing)
OL
UL
valueLIPoorly supported 'counter-reset', allowed in loose, dropped in strict
vspaceIMGNear-equiv styles 'margin-left' and 'margin-right', needs px suffix, see hspace
widthHRNear-equiv style 'width', needs px suffix if original was pixels
TD, TH
+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/dtd/xhtml1-transitional.dtd b/vendor/ezyang/htmlpurifier/docs/dtd/xhtml1-transitional.dtd new file mode 100644 index 0000000000..628f27ac50 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/dtd/xhtml1-transitional.dtd @@ -0,0 +1,1201 @@ + + + + + +%HTMLlat1; + + +%HTMLsymbol; + + +%HTMLspecial; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-customize.html b/vendor/ezyang/htmlpurifier/docs/enduser-customize.html new file mode 100644 index 0000000000..7e1ffa2601 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-customize.html @@ -0,0 +1,850 @@ + + + + + + + +Customize - HTML Purifier + + + +

Customize!

+
HTML Purifier is a Swiss-Army Knife
+ +
Filed under End-User
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

+ HTML Purifier has this quirk where if you try to allow certain elements or + attributes, HTML Purifier will tell you that it's not supported, and that + you should go to the forums to find out how to implement it. Well, this + document is how to implement elements and attributes which HTML Purifier + doesn't support out of the box. +

+ +

Is it necessary?

+ +

+ Before we even write any code, it is paramount to consider whether or + not the code we're writing is necessary or not. HTML Purifier, by default, + contains a large set of elements and attributes: large enough so that + any element or attribute in XHTML 1.0 or 1.1 (and its HTML variants) + that can be safely used by the general public is implemented. +

+ +

+ So what needs to be implemented? (Feel free to skip this section if + you know what you want). +

+ +

XHTML 1.0

+ +

+ All of the modules listed below are based off of the + modularization of + XHTML, which, while technically for XHTML 1.1, is quite a useful + resource. +

+ + + +

+ If you don't recognize it, you probably don't need it. But the curious + can look all of these modules up in the above-mentioned document. Note + that inline scripting comes packaged with HTML Purifier (more on this + later). +

+ +

XHTML 1.1

+ +

+ As of HTMLPurifier 2.1.0, we have implemented the + Ruby module, + which defines a set of tags + for publishing short annotations for text, used mostly in Japanese + and Chinese school texts, but applicable for positioning any text (not + limited to translations) above or below other corresponding text. +

+ +

HTML 5

+ +

+ HTML 5 + is a fork of HTML 4.01 by WHATWG, who believed that XHTML 2.0 was headed + in the wrong direction. It too is a working draft, and may change + drastically before publication, but it should be noted that the + canvas tag has been implemented by many browser vendors. +

+ +

Proprietary

+ +

+ There are a number of proprietary tags still in the wild. Many of them + have been documented in ref-proprietary-tags.txt, + but there is currently no implementation for any of them. +

+ +

Extensions

+ +

+ There are also a number of other XML languages out there that can + be embedded in HTML documents: two of the most popular are MathML and + SVG, and I frequently get requests to implement these. But they are + expansive, comprehensive specifications, and it would take far too long + to implement them correctly (most systems I've seen go as far + as whitelisting tags and no further; come on, what about nesting!) +

+ +

+ Word of warning: HTML Purifier is currently not namespace + aware. +

+ +

Giving back

+ +

+ As you may imagine from the details above (don't be abashed if you didn't + read it all: a glance over would have done), there's quite a bit that + HTML Purifier doesn't implement. Recent architectural changes have + allowed HTML Purifier to implement elements and attributes that are not + safe! Don't worry, they won't be activated unless you set %HTML.Trusted + to true, but they certainly help out users who need to put, say, forms + on their page and don't want to go through the trouble of reading this + and implementing it themself. +

+ +

+ So any of the above that you implement for your own application could + help out some other poor sap on the other side of the globe. Help us + out, and send back code so that it can be hammered into a module and + released with the core. Any code would be greatly appreciated! +

+ +

And now...

+ +

+ Enough philosophical talk, time for some code: +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+if ($def = $config->maybeGetRawHTMLDefinition()) {
+    // our code will go here
+}
+ +

+ Assuming that HTML Purifier has already been properly loaded (hint: + include HTMLPurifier.auto.php), this code will set up + the environment that you need to start customizing the HTML definition. + What's going on? +

+ + + +

Turn off caching

+ +

+ To make development easier, we're going to temporarily turn off + definition caching: +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+$config->set('Cache.DefinitionImpl', null); // TODO: remove this later!
+$def = $config->getHTMLDefinition(true);
+ +

+ A few things should be mentioned about the caching mechanism before + we move on. For performance reasons, HTML Purifier caches generated + HTMLPurifier_Definition objects in serialized files + stored (by default) in library/HTMLPurifier/DefinitionCache/Serializer. + A lot of processing is done in order to create these objects, so it + makes little sense to repeat the same processing over and over again + whenever HTML Purifier is called. +

+ +

+ In order to identify a cache entry, HTML Purifier uses three variables: + the library's version number, the value of %HTML.DefinitionRev and + a serial of relevant configuration. Whenever any of these changes, + a new HTML definition is generated. Notice that there is no way + for the definition object to track changes to customizations: here, it + is up to you to supply appropriate information to DefinitionID and + DefinitionRev. +

+ +

Add an attribute

+ +

+ For this example, we're going to implement the target attribute found + on a elements. To implement an attribute, we have to + ask a few questions: +

+ +
    +
  1. What element is it found on?
    2. +
  2. What is its name?
    3. +
  3. Is it required or optional?
    4. +
  4. What are valid values for it?
    5. +
+ +

+ The first three are easy: the element is a, the attribute + is target, and it is not a required attribute. (If it + was required, we'd need to append an asterisk to the attribute name, + you'll see an example of this in the addElement() example). +

+ +

+ The last question is a little trickier. + Lets allow the special values: _blank, _self, _target and _top. + The form of this is called an enumeration, a list of + valid values, although only one can be used at a time. To translate + this into code form, we write: +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+$config->set('Cache.DefinitionImpl', null); // remove this later!
+$def = $config->getHTMLDefinition(true);
+$def->addAttribute('a', 'target', 'Enum#_blank,_self,_target,_top');
+ +

+ The Enum#_blank,_self,_target,_top does all the magic. + The string is split into two parts, separated by a hash mark (#): +

+ +
    +
  1. The first part is the name of what we call an AttrDef
    2. +
  2. The second part is the parameter of the above-mentioned AttrDef
    3. +
+ +

+ If that sounds vague and generic, it's because it is! HTML Purifier defines + an assortment of different attribute types one can use, and each of these + has their own specialized parameter format. Here are some of the more useful + ones: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeFormatDescription
Enum[s:]value1,value2,... + Attribute with a number of valid values, one of which may be used. When + s: is present, the enumeration is case sensitive. +
Boolattribute_name + Boolean attribute, with only one valid value: the name + of the attribute. +
CDATA + Attribute of arbitrary text. Can also be referred to as Text + (the specification makes a semantic distinction between the two). +
ID + Attribute that specifies a unique ID +
Pixels + Attribute that specifies an integer pixel length +
Length + Attribute that specifies a pixel or percentage length +
NMTOKENS + Attribute that specifies a number of name tokens, example: the + class attribute +
URI + Attribute that specifies a URI, example: the href + attribute +
Number + Attribute that specifies an positive integer number +
+ +

+ For a complete list, consult + library/HTMLPurifier/AttrTypes.php; + more information on attributes that accept parameters can be found on their + respective includes in + library/HTMLPurifier/AttrDef. +

+ +

+ Sometimes, the restrictive list in AttrTypes just doesn't cut it. Don't + sweat: you can also use a fully instantiated object as the value. The + equivalent, verbose form of the above example is: +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+$config->set('Cache.DefinitionImpl', null); // remove this later!
+$def = $config->getHTMLDefinition(true);
+$def->addAttribute('a', 'target', new HTMLPurifier_AttrDef_Enum(
+  array('_blank','_self','_target','_top')
+));
+ +

+ Trust me, you'll learn to love the shorthand. +

+ +

Add an element

+ +

+ Adding attributes is really small-fry stuff, though, and it was possible + to add them (albeit a bit more wordy) prior to 2.0. The real gem of + the Advanced API is adding elements. There are five questions to + ask when adding a new element: +

+ +
    +
  1. What is the element's name?
    2. +
  2. What content set does this element belong to?
    3. +
  3. What are the allowed children of this element?
    4. +
  4. What attributes does the element allow that are general?
    5. +
  5. What attributes does the element allow that are specific to this element?
    6. +
+ +

+ It's a mouthful, and you'll be slightly lost if your not familiar with + the HTML specification, so let's explain them step by step. +

+ +

Content set

+ +

+ The HTML specification defines two major content sets: Inline + and Block. Each of these + content sets contain a list of elements: Inline contains things like + span and b while Block contains things like + div and blockquote. +

+ +

+ These content sets amount to a macro mechanism for HTML definition. Most + elements in HTML are organized into one of these two sets, and most + elements in HTML allow elements from one of these sets. If we had + to write each element verbatim into each other element's allowed + children, we would have ridiculously large lists; instead we use + content sets to compactify the declaration. +

+ +

+ Practically speaking, there are several useful values you can use here: +

+ + + + + + + + + + + + + + + + + + + + + + +
Content setDescription
InlineCharacter level elements, text
BlockBlock-like elements, like paragraphs and lists
false + Any element that doesn't fit into the mold, for example li + or tr +
+ +

+ By specifying a valid value here, all other elements that use that + content set will also allow your element, without you having to do + anything. If you specify false, you'll have to register + your element manually. +

+ +

Allowed children

+ +

+ Allowed children defines the elements that this element can contain. + The allowed values may range from none to a complex regexp depending on + your element. +

+ +

+ If you've ever taken a look at the HTML DTD's before, you may have + noticed declarations like this: +

+ +
<!ELEMENT LI - O (%flow;)*             -- list item -->
+ +

+ The (%flow;)* indicates the allowed children of the + li tag: li allows any number of flow + elements as its children. (The - O allows the closing tag to be + omitted, though in XML this is not allowed.) In HTML Purifier, + we'd write it like Flow (here's where the content sets + we were discussing earlier come into play). There are three shorthand + content models you can specify: +

+ + + + + + + + + + + + + + + + + + + + + + +
Content modelDescription
EmptyNo children allowed, like br or hr
InlineAny number of inline elements and text, like span
FlowAny number of inline elements, block elements and text, like div
+ +

+ This covers 90% of all the cases out there, but what about elements that + break the mold like ul? This guy requires at least one + child, and the only valid children for it are li. The + content model is: Required: li. There are two parts: the + first type determines what ChildDef will be used to validate + content models. The most common values are: +

+ + + + + + + + + + + + + + + + + + + + + + +
TypeDescription
RequiredChildren must be one or more of the valid elements
OptionalChildren can be any number of the valid elements
CustomChildren must follow the DTD-style regex
+ +

+ You can also implement your own ChildDef: this was done + for a few special cases in HTML Purifier such as Chameleon + (for ins and del), StrictBlockquote + and Table. +

+ +

+ The second part specifies either valid elements or a regular expression. + Valid elements are separated with horizontal bars (|), i.e. + "a | b | c". Use #PCDATA to represent plain text. + Regular expressions are based off of DTD's style: +

+ + + +

+ For example, "a, b?, (c | d), e+, f*" means "In this order, + one a element, at most one b element, + one c or d element (but not both), one or more + e elements, and any number of f elements." + Regex veterans should be able to jump right in, and those not so savvy + can always copy-paste W3C's content model definitions into HTML Purifier + and hope for the best. +

+ +

+ A word of warning: while the regex format is extremely flexible on + the developer's side, it is + quite unforgiving on the user's side. If the user input does not exactly + match the specification, the entire contents of the element will + be nuked. This is why there is are specific content model types like + Optional and Required: while they could be implemented as Custom: + (valid | elements)*, the custom classes contain special recovery + measures that make sure as much of the user's original content gets + through. HTML Purifier's core, as a rule, does not use Custom. +

+ +

+ One final note: you can also use Content Sets inside your valid elements + lists or regular expressions. In fact, the three shorthand content models + mentioned above are just that: abbreviations: +

+ + + + + + + + + + + + + + + + + + +
Content modelImplementation
InlineOptional: Inline | #PCDATA
FlowOptional: Flow | #PCDATA
+ +

+ When the definition is compiled, Inline will be replaced with a + horizontal-bar separated list of inline elements. Also, notice that + it does not contain text: you have to specify that yourself. +

+ +

Common attributes

+ +

+ Congratulations: you have just gotten over the proverbial hump (Allowed + children). Common attributes is much simpler, and boils down to + one question: does your element have the id, style, + class, title and lang attributes? + If so, you'll want to specify the Common attribute collection, + which contains these five attributes that are found on almost every + HTML element in the specification. +

+ +

+ There are a few more collections, but they're really edge cases: +

+ + + + + + + + + + + + + + + + + + +
CollectionAttributes
I18Nlang, possibly xml:lang
Corestyle, class, id and title
+ +

+ Common is a combination of the above-mentioned collections. +

+ +

+ Readers familiar with the modularization may have noticed that the Core + attribute collection differs from that specified by the abstract + modules of the XHTML Modularization 1.1. We believe this section + to be in error, as br permits the use of the style + attribute even though it uses the Core collection, and + the DTD and XML Schemas supplied by W3C support our interpretation. +

+ +

Attributes

+ +

+ If you didn't read the earlier section on + adding attributes, read it now. The last parameter is simply + an array of attribute names to attribute implementations, in the exact + same format as addAttribute(). +

+ +

Putting it all together

+ +

+ We're going to implement form. Before we embark, lets + grab a reference implementation from over at the + transitional DTD: +

+ +
<!ELEMENT FORM - - (%flow;)* -(FORM)   -- interactive form -->
+<!ATTLIST FORM
+  %attrs;                              -- %coreattrs, %i18n, %events --
+  action      %URI;          #REQUIRED -- server-side form handler --
+  method      (GET|POST)     GET       -- HTTP method used to submit the form--
+  enctype     %ContentType;  "application/x-www-form-urlencoded"
+  accept      %ContentTypes; #IMPLIED  -- list of MIME types for file upload --
+  name        CDATA          #IMPLIED  -- name of form for scripting --
+  onsubmit    %Script;       #IMPLIED  -- the form was submitted --
+  onreset     %Script;       #IMPLIED  -- the form was reset --
+  target      %FrameTarget;  #IMPLIED  -- render in this frame --
+  accept-charset %Charsets;  #IMPLIED  -- list of supported charsets --
+  >
+ +

+ Juicy! With just this, we can answer four of our five questions: +

+ +
    +
  1. What is the element's name? form
    2. +
  2. What content set does this element belong to? Block + (this needs a little sleuthing, I find the easiest way is to search + the DTD for FORM and determine which set it is in.)
    3. +
  3. What are the allowed children of this element? One + or more flow elements, but no nested forms
    4. +
  4. What attributes does the element allow that are general? Common
    5. +
  5. What attributes does the element allow that are specific to this element? A whole bunch, see ATTLIST; + we're going to do the vital ones: action, method and name
    6. +
+ +

+ Time for some code: +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+$config->set('Cache.DefinitionImpl', null); // remove this later!
+$def = $config->getHTMLDefinition(true);
+$def->addAttribute('a', 'target', new HTMLPurifier_AttrDef_Enum(
+  array('_blank','_self','_target','_top')
+));
+$form = $def->addElement(
+  'form',   // name
+  'Block',  // content set
+  'Flow', // allowed children
+  'Common', // attribute collection
+  array( // attributes
+    'action*' => 'URI',
+    'method' => 'Enum#get|post',
+    'name' => 'ID'
+  )
+);
+$form->excludes = array('form' => true);
+ +

+ Each of the parameters corresponds to one of the questions we asked. + Notice that we added an asterisk to the end of the action + attribute to indicate that it is required. If someone specifies a + form without that attribute, the tag will be axed. + Also, the extra line at the end is a special extra declaration that + prevents forms from being nested within each other. +

+ +

+ And that's all there is to it! Implementing the rest of the form + module is left as an exercise to the user; to see more examples + check the library/HTMLPurifier/HTMLModule/ directory + in your local HTML Purifier installation. +

+ +

And beyond...

+ +

+ Perceptive users may have realized that, to a certain extent, we + have simply re-implemented the facilities of XML Schema or the + Document Type Definition. What you are seeing here, however, is + not just an XML Schema or Document Type Definition: it is a fully + expressive method of specifying the definition of HTML that is + a portable superset of the capabilities of the two above-mentioned schema + languages. What makes HTMLDefinition so powerful is the fact that + if we don't have an implementation for a content model or an attribute + definition, you can supply it yourself by writing a PHP class. +

+ +

+ There are many facets of HTMLDefinition beyond the Advanced API I have + walked you through today. To find out more about these, you can + check out these source files: +

+ + + +

Notes for HTML Purifier 4.2.0 and earlier

+ +

+ Previously, this tutorial gave some incorrect template code for + editing raw definitions, and that template code will now produce the + error Due to a documentation error in previous version of HTML + Purifier... Here is how to mechanically transform old-style + code into new-style code. +

+ +

+ First, identify all code that edits the raw definition object, and + put it together. Ensure none of this code must be run on every + request; if some sub-part needs to always be run, move it outside + this block. Here is an example below, with the raw definition + object code bolded. +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+$def = $config->getHTMLDefinition(true);
+$def->addAttribute('a', 'target', 'Enum#_blank,_self,_target,_top');
+$purifier = new HTMLPurifier($config);
+ +

+ Next, replace the raw definition retrieval with a + maybeGetRawHTMLDefinition method call inside an if conditional, and + place the editing code inside that if block. +

+ +
$config = HTMLPurifier_Config::createDefault();
+$config->set('HTML.DefinitionID', 'enduser-customize.html tutorial');
+$config->set('HTML.DefinitionRev', 1);
+if ($def = $config->maybeGetRawHTMLDefinition()) {
+    $def->addAttribute('a', 'target', 'Enum#_blank,_self,_target,_top');
+}
+$purifier = new HTMLPurifier($config);
+ +

+ And you're done! Alternatively, if you're OK with not ever caching + your code, the following will still work and not emit warnings. +

+ +
$config = HTMLPurifier_Config::createDefault();
+$def = $config->getHTMLDefinition(true);
+$def->addAttribute('a', 'target', 'Enum#_blank,_self,_target,_top');
+$purifier = new HTMLPurifier($config);
+ +

+ A slightly less efficient version of this was what was going on with + old versions of HTML Purifier. +

+ +

+ Technical notes: ajh pointed out on in a forum topic that + HTML Purifier appeared to be repeatedly writing to the cache even + when a cache entry already existed. Investigation lead to the + discovery of the following infelicity: caching of customized + definitions didn't actually work! The problem was that even though + a cache file would be written out at the end of the process, there + was no way for HTML Purifier to say, Actually, I've already got a + copy of your work, no need to reconfigure your + customizations. This required the API to change: placing + all of the customizations to the raw definition object in a + conditional which could be skipped. +

+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-id.html b/vendor/ezyang/htmlpurifier/docs/enduser-id.html new file mode 100644 index 0000000000..53d2da2489 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-id.html @@ -0,0 +1,148 @@ + + + + + + + +IDs - HTML Purifier + + + +

IDs

+
What they are, why you should(n't) wear them, and how to deal with it
+ +
Filed under End-User
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

Prior to HTML Purifier 1.2.0, this library blithely accepted user input that +looked like this:

+ +
<a id="fragment">Anchor</a>
+ +

...presenting an attractive vector for those that would destroy standards +compliance: simply set the ID to one that is already used elsewhere in the +document and voila: validation breaks. There was a half-hearted attempt to +prevent this by allowing users to blacklist IDs, but I suspect that no one +really bothered, and thus, with the release of 1.2.0, IDs are now removed +by default.

+ +

IDs, however, are quite useful functionality to have, so if users start +complaining about broken anchors you'll probably want to turn them back on +with %Attr.EnableID. But before you go mucking around with the config +object, it's probably worth to take some precautions to keep your page +validating. Why?

+ +
    +
  1. Standards-compliant pages are good
    2. +
  2. Duplicated IDs interfere with anchors. If there are two id="foobar"s in a + document, which spot does a browser presented with the fragment #foobar go + to? Most browsers opt for the first appearing ID, making it impossible + to references the second section. Similarly, duplicated IDs can hijack + client-side scripting that relies on the IDs of elements.
    3. +
+ +

You have (currently) four ways of dealing with the problem.

+ + + +

Blacklisting IDs

+
Good for pages with single content source and stable templates
+ +

Keeping in terms with the +KISS principle, let us +deal with the most obvious solution: preventing users from using any IDs that +appear elsewhere on the document. The method is simple:

+ +
$config->set('Attr.EnableID', true);
+$config->set('Attr.IDBlacklist' array(
+    'list', 'of', 'attribute', 'values', 'that', 'are', 'forbidden'
+));
+ +

That being said, there are some notable drawbacks. First of all, you have to +know precisely which IDs are being used by the HTML surrounding the user code. +This is easier said than done: quite often the page designer and the system +coder work separately, so the designer has to constantly be talking with the +coder whenever he decides to add a new anchor. Miss one and you open yourself +to possible standards-compliance issues.

+ +

Furthermore, this position becomes untenable when a single web page must hold +multiple portions of user-submitted content. Since there's obviously no way +to find out before-hand what IDs users will use, the blacklist is helpless. +And since HTML Purifier validates each segment separately, perhaps doing +so at different times, it would be extremely difficult to dynamically update +the blacklist in between runs.

+ +

Finally, simply destroying the ID is extremely un-userfriendly behavior: after +all, they might have simply specified a duplicate ID by accident.

+ +

Thus, we get to our second method.

+ + + +

Namespacing IDs

+
Lazy developer's way, but needs user education
+ +

This method, too, is quite simple: add a prefix to all user IDs. With this +code:

+ +
$config->set('Attr.EnableID', true);
+$config->set('Attr.IDPrefix', 'user_');
+ +

...this:

+ +
<a id="foobar">Anchor!</a>
+ +

...turns into:

+ +
<a id="user_foobar">Anchor!</a>
+ +

As long as you don't have any IDs that start with user_, collisions are +guaranteed not to happen. The drawback is obvious: if a user submits +id="foobar", they probably expect to be able to reference their page with +#foobar. You'll have to tell them, "No, that doesn't work, you have to add +user_ to the beginning."

+ +

And yes, things get hairier. Even with a nice prefix, we still have done +nothing about multiple HTML Purifier outputs on one page. Thus, we have +a second configuration value to piggy-back off of: %Attr.IDPrefixLocal:

+ +
$config->set('Attr.IDPrefixLocal', 'comment' . $id . '_');
+ +

This new attributes does nothing but append on to regular IDPrefix, but is +special in that it is volatile: it's value is determined at run-time and +cannot possibly be cordoned into, say, a .ini config file. As for what to +put into the directive, is up to you, but I would recommend the ID number +the text has been assigned in the database. Whatever you pick, however, it +has to be unique and stable for the text you are validating. Note, however, +that we require that %Attr.IDPrefix be set before you use this directive.

+ +

And also remember: the user has to know what this prefix is too!

+ + + +

Abstinence

+ +

You may not want to bother. That's okay too, just don't enable IDs.

+ +

Personally, I would take this road whenever user-submitted content would be +possibly be shown together on one page. Why a blog comment would need to use +anchors is beyond me.

+ + + +

Denial

+ +

To revert back to pre-1.2.0 behavior, simply:

+ +
$config->set('Attr.EnableID', true);
+ +

Don't come crying to me when your page mysteriously stops validating, though.

+ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-overview.txt b/vendor/ezyang/htmlpurifier/docs/enduser-overview.txt new file mode 100644 index 0000000000..fe7f8705dd --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-overview.txt @@ -0,0 +1,59 @@ + +HTML Purifier + by Edward Z. Yang + +There are a number of ad hoc HTML filtering solutions out there on the web +(some examples including HTML_Safe, kses and SafeHtmlChecker.class.php) that +claim to filter HTML properly, preventing malicious JavaScript and layout +breaking HTML from getting through the parser. None of them, however, +demonstrates a thorough knowledge of neither the DTD that defines the HTML +nor the caveats of HTML that cannot be expressed by a DTD. Configurable +filters (such as kses or PHP's built-in striptags() function) have trouble +validating the contents of attributes and can be subject to security attacks +due to poor configuration. Other filters take the naive approach of +blacklisting known threats and tags, failing to account for the introduction +of new technologies, new tags, new attributes or quirky browser behavior. + +However, HTML Purifier takes a different approach, one that doesn't use +specification-ignorant regexes or narrow blacklists. HTML Purifier will +decompose the whole document into tokens, and rigorously process the tokens by: +removing non-whitelisted elements, transforming bad practice tags like +into , properly checking the nesting of tags and their children and +validating all attributes according to their RFCs. + +To my knowledge, there is nothing like this on the web yet. Not even MediaWiki, +which allows an amazingly diverse mix of HTML and wikitext in its documents, +gets all the nesting quirks right. Existing solutions hope that no JavaScript +will slip through, but either do not attempt to ensure that the resulting +output is valid XHTML or send the HTML through a draconic XML parser (and yet +still get the nesting wrong: SafeHtmlChecker.class.php does not prevent +tags from being nested within each other). + +This document no longer is a detailed description of how HTMLPurifier works, +as those descriptions have been moved to the appropriate code. The first +draft was drawn up after two rough code sketches and the implementation of a +forgiving lexer. You may also be interested in the unit tests located in the +tests/ folder, which provide a living document on how exactly the filter deals +with malformed input. + +In summary (see corresponding classes for more details): + +1. Parse document into an array of tag and text tokens (Lexer) +2. Remove all elements not on whitelist and transform certain other elements + into acceptable forms (i.e. ) +3. Make document well formed while helpfully taking into account certain quirks, + such as the fact that

tags traditionally are closed by other block-level + elements. +4. Run through all nodes and check children for proper order (especially + important for tables). +5. Validate attributes according to more restrictive definitions based on the + RFCs. +6. Translate back into a string. (Generator) + +HTML Purifier is best suited for documents that require a rich array of +HTML tags. Things like blog comments are, in all likelihood, most appropriately +written in an extremely restrictive set of markup that doesn't require +all this functionality (or not written in HTML at all), although this may +be changing in the future with the addition of levels of filtering. + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-security.txt b/vendor/ezyang/htmlpurifier/docs/enduser-security.txt new file mode 100644 index 0000000000..518f092bd6 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-security.txt @@ -0,0 +1,18 @@ + +Security + +Like anything that claims to afford security, HTML_Purifier can be circumvented +through negligence of people. This class will do its job: no more, no less, +and it's up to you to provide it the proper information and proper context +to be effective. Things to remember: + +1. Character Encoding: see enduser-utf8.html for more info. + +2. IDs: see enduser-id.html for more info + +3. URIs: see enduser-uri-filter.html + +4. CSS: document pending +Explain which CSS styles we blocked and why. + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-slow.html b/vendor/ezyang/htmlpurifier/docs/enduser-slow.html new file mode 100644 index 0000000000..f0ea02de1c --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-slow.html @@ -0,0 +1,120 @@ + + + + + + + +Speeding up HTML Purifier - HTML Purifier + + + +

Speeding up HTML Purifier

+
...also known as the HELP ME LIBRARY IS TOO SLOW MY PAGE TAKE TOO LONG page
+ +
Filed under End-User
+ +
HTML Purifier End-User Documentation
+ +

HTML Purifier is a very powerful library. But with power comes great +responsibility, in the form of longer execution times. Remember, this +library isn't lightly grazing over submitted HTML: it's deconstructing +the whole thing, rigorously checking the parts, and then putting it back +together.

+ +

So, if it so turns out that HTML Purifier is kinda too slow for outbound +filtering, you've got a few options:

+ +

Inbound filtering

+ +

Perform filtering of HTML when it's submitted by the user. Since the +user is already submitting something, an extra half a second tacked on +to the load time probably isn't going to be that huge of a problem. +Then, displaying the content is a simple a manner of outputting it +directly from your database/filesystem. The trouble with this method is +that your user loses the original text, and when doing edits, will be +handling the filtered text. While this may be a good thing, especially +if you're using a WYSIWYG editor, it can also result in data-loss if a +user makes a typo.

+ +

Example (non-functional):

+ +
<?php
+    /**
+     * FORM SUBMISSION PAGE
+     * display_error($message) : displays nice error page with message
+     * display_success() : displays a nice success page
+     * display_form() : displays the HTML submission form
+     * database_insert($html) : inserts data into database as new row
+     */
+    if (!empty($_POST)) {
+        require_once '/path/to/library/HTMLPurifier.auto.php';
+        require_once 'HTMLPurifier.func.php';
+        $dirty_html = isset($_POST['html']) ? $_POST['html'] : false;
+        if (!$dirty_html) {
+            display_error('You must write some HTML!');
+        }
+        $html = HTMLPurifier($dirty_html);
+        database_insert($html);
+        display_success();
+        // notice that $dirty_html is *not* saved
+    } else {
+        display_form();
+    }
+?>
+ +

Caching the filtered output

+ +

Accept the submitted text and put it unaltered into the database, but +then also generate a filtered version and stash that in the database. +Serve the filtered version to readers, and the unaltered version to +editors. If need be, you can invalidate the cache and have the cached +filtered version be regenerated on the first page view. Pros? Full data +retention. Cons? It's more complicated, and opens other editors up to +XSS if they are using a WYSIWYG editor (to fix that, they'd have to be +able to get their hands on the *really* original text served in +plaintext mode).

+ +

Example (non-functional):

+ +
<?php
+    /**
+     * VIEW PAGE
+     * display_error($message) : displays nice error page with message
+     * cache_get($id) : retrieves HTML from fast cache (db or file)
+     * cache_insert($id, $html) : inserts good HTML into cache system
+     * database_get($id) : retrieves raw HTML from database
+     */
+    $id = isset($_GET['id']) ? (int) $_GET['id'] : false;
+    if (!$id) {
+        display_error('Must specify ID.');
+        exit;
+    }
+    $html = cache_get($id); // filesystem or database
+    if ($html === false) {
+        // cache didn't have the HTML, generate it
+        $raw_html = database_get($id);
+        require_once '/path/to/library/HTMLPurifier.auto.php';
+        require_once 'HTMLPurifier.func.php';
+        $html = HTMLPurifier($raw_html);
+        cache_insert($id, $html);
+    }
+    echo $html;
+?>
+ +

Summary

+ +

In short, inbound filtering is the simple option and caching is the +robust option (albeit with bigger storage requirements).

+ +

There is a third option, independent of the two we've discussed: profile +and optimize HTMLPurifier yourself. Be sure to report back your results +if you decide to do that! Especially if you port HTML Purifier to C++. +;-)

+ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-tidy.html b/vendor/ezyang/htmlpurifier/docs/enduser-tidy.html new file mode 100644 index 0000000000..a243f7fc23 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-tidy.html @@ -0,0 +1,231 @@ + + + + + + + +Tidy - HTML Purifier + + + +

Tidy

+ +
Filed under Development
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

You've probably heard of HTML Tidy, Dave Raggett's little piece +of software that cleans up poorly written HTML. Let me say it straight +out:

+ +

This ain't HTML Tidy!

+ +

Rather, Tidy stands for a cool set of Tidy-inspired features in HTML Purifier +that allows users to submit deprecated elements and attributes and get +valid strict markup back. For example:

+ +
<center>Centered</center>
+ +

...becomes:

+ +
<div style="text-align:center;">Centered</div>
+ +

...when this particular fix is run on the HTML. This tutorial will give +you the lowdown of what exactly HTML Purifier will do when Tidy +is on, and how to fine-tune this behavior. Once again, you do +not need Tidy installed on your PHP to use these features!

+ +

What does it do?

+ +

Tidy will do several things to your HTML:

+ +
    +
  • Convert deprecated elements and attributes to standards-compliant + alternatives
    • +
  • Enforce XHTML compatibility guidelines and other best practices
    • +
  • Preserve data that would normally be removed as per W3C
    • +
+ +

What are levels?

+ +

Levels describe how aggressive the Tidy module should be when +cleaning up HTML. There are four levels to pick: none, light, medium +and heavy. Each of these levels has a well-defined set of behavior +associated with it, although it may change depending on your doctype.

+ +
+
light
+
This is the lenient level. If a tag or attribute + is about to be removed because it isn't supported by the + doctype, Tidy will step in and change into an alternative that + is supported.
+
medium
+
This is the correctional level. At this level, + all the functions of light are performed, as well as some extra, + non-essential best practices enforcement. Changes made on this + level are very benign and are unlikely to cause problems.
+
heavy
+
This is the aggressive level. If a tag or + attribute is deprecated, it will be converted into a non-deprecated + version, no ifs ands or buts.
+
+ +

By default, Tidy operates on the medium level. You can +change the level of cleaning by setting the %HTML.TidyLevel configuration +directive:

+ +
$config->set('HTML.TidyLevel', 'heavy'); // burn baby burn!
+ +

Is the light level really light?

+ +

It depends on what doctype you're using. If your documents are HTML +4.01 Transitional, HTML Purifier will be lazy +and won't clean up your center +or font tags. But if you're using HTML 4.01 Strict, +HTML Purifier has no choice: it has to convert them, or they will +be nuked out of existence. So while light on Transitional will result +in little to no changes, light on Strict will still result in quite +a lot of fixes.

+ +

This is different behavior from 1.6 or before, where deprecated +tags in transitional documents would +always be cleaned up regardless. This is also better behavior.

+ +

My pages look different!

+ +

HTML Purifier is tasked with converting deprecated tags and +attributes to standards-compliant alternatives, which usually +need copious amounts of CSS. It's also not foolproof: sometimes +things do get lost in the translation. This is why when HTML Purifier +can get away with not doing cleaning, it won't; this is why +the default value is medium and not heavy.

+ +

Fortunately, only a few attributes have problems with the switch +over. They are described below:

+ + + + + + + + + + + + + + + + + + + + + + + + +
Element@AttrChanges
caption@alignFirefox supports stuffing the caption on the + left and right side of the table, a feature that + Internet Explorer, understandably, does not have. + When align equals right or left, the text will simply + be aligned on the left or right side.
img@alignThe implementation for align bottom is good, but not + perfect. There are a few pixel differences.
br@clearClear both gets a little wonky in Internet Explorer. Haven't + really been able to figure out why.
hr@noshadeAll browsers implement this slightly differently: we've + chosen to make noshade horizontal rules gray.
+ +

There are a few more minor, although irritating, bugs. +Some older browsers support deprecated attributes, +but not CSS. Transformed elements and attributes will look unstyled +to said browsers. Also, CSS precedence is slightly different for +inline styles versus presentational markup. In increasing precedence:

+ +
    +
  1. Presentational attributes
    2. +
  2. External style sheets
    3. +
  3. Inline styling
    4. +
+ +

This means that styling that may have been masked by external CSS +declarations will start showing up (a good thing, perhaps). Finally, +if you've turned off the style attribute, almost all of +these transformations will not work. Sorry mates.

+ +

You can review the rendering before and after of these transformations +by consulting the attrTransform.php +smoketest.

+ +

I like the general idea, but the specifics bug me!

+ +

So you want HTML Purifier to clean up your HTML, but you're not +so happy about the br@clear implementation. That's perfectly fine! +HTML Purifier will make accomodations:

+ +
$config->set('HTML.Doctype', 'XHTML 1.0 Transitional');
+$config->set('HTML.TidyLevel', 'heavy'); // all changes, minus...
+$config->set('HTML.TidyRemove', 'br@clear');
+ +

That third line does the magic, removing the br@clear fix +from the module, ensuring that <br clear="both" /> +will pass through unharmed. The reverse is possible too:

+ +
$config->set('HTML.Doctype', 'XHTML 1.0 Transitional');
+$config->set('HTML.TidyLevel', 'none'); // no changes, plus...
+$config->set('HTML.TidyAdd', 'p@align');
+ +

In this case, all transformations are shut off, except for the p@align +one, which you found handy.

+ +

To find out what the names of fixes you want to turn on or off are, +you'll have to consult the source code, specifically the files in +HTMLPurifier/HTMLModule/Tidy/. There is, however, a +general syntax:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameExampleInterpretation
elementfontTag transform for element
element@attrbr@clearAttribute transform for attr on element
@attr@langGlobal attribute transform for attr
e#content_model_typeblockquote#content_model_typeChange of child processing implementation for e
+ +

So... what's the lowdown?

+ +

The lowdown is, quite frankly, HTML Purifier's default settings are +probably good enough. The next step is to bump the level up to heavy, +and if that still doesn't satisfy your appetite, do some fine-tuning. +Other than that, don't worry about it: this all works silently and +effectively in the background.

+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-uri-filter.html b/vendor/ezyang/htmlpurifier/docs/enduser-uri-filter.html new file mode 100644 index 0000000000..d1b3354a35 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-uri-filter.html @@ -0,0 +1,204 @@ + + + + + + + +URI Filters - HTML Purifier + + + +

URI Filters

+ +
Filed under End-User
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

+ This is a quick and dirty document to get you on your way to writing + custom URI filters for your own URL filtering needs. Why would you + want to write a URI filter? If you need URIs your users put into + HTML to magically change into a different URI, this is + exactly what you need! +

+ +

Creating the class

+ +

+ Any URI filter you make will be a subclass of HTMLPurifier_URIFilter. + The scaffolding is thus: +

+ +
class HTMLPurifier_URIFilter_NameOfFilter extends HTMLPurifier_URIFilter
+{
+    public $name = 'NameOfFilter';
+    public function prepare($config) {}
+    public function filter(&$uri, $config, $context) {}
+}
+ +

+ Fill in the variable $name with the name of your filter, and + take a look at the two methods. prepare() is an initialization + method that is called only once, before any filtering has been done of the + HTML. Use it to perform any costly setup work that only needs to be done + once. filter() is the guts and innards of our filter: + it takes the URI and does whatever needs to be done to it. +

+ +

+ If you've worked with HTML Purifier, you'll recognize the $config + and $context parameters. On the other hand, $uri + is something unique to this section of the application: it's a + HTMLPurifier_URI object. The interface is thus: +

+ +
class HTMLPurifier_URI
+{
+    public $scheme, $userinfo, $host, $port, $path, $query, $fragment;
+    public function HTMLPurifier_URI($scheme, $userinfo, $host, $port, $path, $query, $fragment);
+    public function toString();
+    public function copy();
+    public function getSchemeObj($config, $context);
+    public function validate($config, $context);
+}
+ +

+ The first three methods are fairly self-explanatory: you have a constructor, + a serializer, and a cloner. Generally, you won't be using them when + you are manipulating the URI objects themselves. + getSchemeObj() is a special purpose method that returns + a HTMLPurifier_URIScheme object corresponding to the specific + URI at hand. validate() performs general-purpose validation + on the internal components of a URI. Once again, you don't need to + worry about these: they've already been handled for you. +

+ +

URI format

+ +

+ As a URIFilter, we're interested in the member variables of the URI object. +

+ + + + + + + + + +
Scheme The protocol for identifying (and possibly locating) a resource (http, ftp, https)
Userinfo User information such as a username (bob)
Host Domain name or IP address of the server (example.com, 127.0.0.1)
Port Network port number for the server (80, 12345)
Path Data that identifies the resource, possibly hierarchical (/path/to, ed@example.com)
Query String of information to be interpreted by the resource (?q=search-term)
Fragment Additional information for the resource after retrieval (#bookmark)
+ +

+ Because the URI is presented to us in this form, and not + http://bob@example.com:8080/foo.php?q=string#hash, it saves us + a lot of trouble in having to parse the URI every time we want to filter + it. For the record, the above URI has the following components: +

+ + + + + + + + + +
Scheme http
Userinfo bob
Host example.com
Port 8080
Path /foo.php
Query q=string
Fragment hash
+ +

+ Note that there is no question mark or octothorpe in the query or + fragment: these get removed during parsing. +

+ +

+ With this information, you can get straight to implementing your + filter() method. But one more thing... +

+ +

Return value: Boolean, not URI

+ +

+ You may have noticed that the URI is being passed in by reference. + This means that whatever changes you make to it, those changes will + be reflected in the URI object the callee had. Do not + return the URI object: it is unnecessary and will cause bugs. + Instead, return a boolean value, true if the filtering was successful, + or false if the URI is beyond repair and needs to be axed. +

+ +

+ Let's suppose I wanted to write a filter that converted links with a + custom image scheme to its corresponding real path on + our website: +

+ +
class HTMLPurifier_URIFilter_TransformImageScheme extends HTMLPurifier_URIFilter
+{
+    public $name = 'TransformImageScheme';
+    public function filter(&$uri, $config, $context) {
+        if ($uri->scheme !== 'image') return true;
+        $img_name = $uri->path;
+        // Overwrite the previous URI object
+        $uri = new HTMLPurifier_URI('http', null, null, null, '/img/' . $img_name . '.png', null, null);
+        return true;
+    }
+}
+ +

+ Notice I did not return $uri;. This filter would turn + image:Foo into /img/Foo.png. +

+ +

Activating your filter

+ +

+ Having a filter is all well and good, but you need to tell HTML Purifier + to use it. Fortunately, this part's simple: +

+ +
$uri = $config->getDefinition('URI');
+$uri->addFilter(new HTMLPurifier_URIFilter_NameOfFilter(), $config);
+ +

+ After adding a filter, you won't be able to set configuration directives. + Structure your code accordingly. +

+ + + +

Post-filter

+ +

+ Remember our TransformImageScheme filter? That filter acted before we had + performed scheme validation; otherwise, the URI would have been filtered + out when it was discovered that there was no image scheme. Well, a post-filter + is run after scheme specific validation, so it's ideal for bulk + post-processing of URIs, including munging. To specify a URI as a post-filter, + set the $post member variable to TRUE. +

+ +
class HTMLPurifier_URIFilter_MyPostFilter extends HTMLPurifier_URIFilter
+{
+    public $name = 'MyPostFilter';
+    public $post = true;
+    // ... extra code here
+}
+
+ +

Examples

+ +

+ Check the + URIFilter + directory for more implementation examples, and see the + new directives proposal document for ideas on what could be implemented + as a filter. +

+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-utf8.html b/vendor/ezyang/htmlpurifier/docs/enduser-utf8.html new file mode 100644 index 0000000000..9b01a302a6 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-utf8.html @@ -0,0 +1,1060 @@ + + + + + + + + +UTF-8: The Secret of Character Encoding - HTML Purifier + + + + + +

UTF-8: The Secret of Character Encoding

+ +
Filed under End-User
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

Character encoding and character sets are not that +difficult to understand, but so many people blithely stumble +through the worlds of programming without knowing what to actually +do about it, or say "Ah, it's a job for those internationalization +experts." No, it is not! This document will walk you through +determining the encoding of your system and how you should handle +this information. It will stay away from excessive discussion on +the internals of character encoding.

+ +

This document is not designed to be read in its entirety: it will +slowly introduce concepts that build on each other: you need not get to +the bottom to have learned something new. However, I strongly +recommend you read all the way to Why UTF-8?, because at least +at that point you'd have made a conscious decision not to migrate, +which can be a rewarding (but difficult) task.

+ +
+
Asides
+

Text in this formatting is an aside, + interesting tidbits for the curious but not strictly necessary material to + do the tutorial. If you read this text, you'll come out + with a greater understanding of the underlying issues.

+
+ +

Table of Contents

+ +
    +
  1. Finding the real encoding
    2. +
  2. Finding the embedded encoding
    3. +
  3. Fixing the encoding
      +
    1. No embedded encoding
      2. +
    2. Embedded encoding disagrees
      3. +
    3. Changing the server encoding
        +
      1. PHP header() function
        2. +
      2. PHP ini directive
        3. +
      3. Non-PHP
        4. +
      4. .htaccess
        5. +
      5. File extensions
        6. +
      4. +
    4. XML
      5. +
    5. Inside the process
      6. +
    4. +
  4. Why UTF-8?
      +
    1. Internationalization
      2. +
    2. User-friendly
      3. +
    3. Forms
        +
      1. application/x-www-form-urlencoded
        2. +
      2. multipart/form-data
        3. +
      4. +
    4. Well supported
      5. +
    5. HTML Purifiers
      6. +
    5. +
  5. Migrate to UTF-8
      +
    1. Configuring your database
        +
      1. Legit method
        2. +
      2. Binary
        3. +
      2. +
    2. Text editor
      3. +
    3. Byte Order Mark (headers already sent!)
      4. +
    4. Fonts
        +
      1. Obscure scripts
        2. +
      2. Occasional use
        3. +
      5. +
    5. Dealing with variable width in functions
      6. +
    6. +
  6. Further Reading
    7. +
+ +

Finding the real encoding

+ +

In the beginning, there was ASCII, and things were simple. But they +weren't good, for no one could write in Cyrillic or Thai. So there +exploded a proliferation of character encodings to remedy the problem +by extending the characters ASCII could express. This ridiculously +simplified version of the history of character encodings shows us that +there are now many character encodings floating around.

+ +
+

A character encoding tells the computer how to + interpret raw zeroes and ones into real characters. It + usually does this by pairing numbers with characters.

+

There are many different types of character encodings floating + around, but the ones we deal most frequently with are ASCII, + 8-bit encodings, and Unicode-based encodings.

+
    +
  • ASCII is a 7-bit encoding based on the + English alphabet.
    • +
  • 8-bit encodings are extensions to ASCII + that add a potpourri of useful, non-standard characters + like é and æ. They can only add 127 characters, + so usually only support one script at a time. When you + see a page on the web, chances are it's encoded in one + of these encodings.
    • +
  • Unicode-based encodings implement the + Unicode standard and include UTF-8, UTF-16 and UTF-32/UCS-4. + They go beyond 8-bits and support almost + every language in the world. UTF-8 is gaining traction + as the dominant international encoding of the web.
    • +
+
+ +

The first step of our journey is to find out what the encoding of +your website is. The most reliable way is to ask your +browser:

+ +
+
Mozilla Firefox
+
Tools > Page Info: Encoding
+
Internet Explorer
+
View > Encoding: bulleted item is unofficial name
+
+ +

Internet Explorer won't give you the MIME (i.e. useful/real) name of the +character encoding, so you'll have to look it up using their description. +Some common ones:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IE's DescriptionMime Name
Windows
Arabic (Windows)Windows-1256
Baltic (Windows)Windows-1257
Central European (Windows)Windows-1250
Cyrillic (Windows)Windows-1251
Greek (Windows)Windows-1253
Hebrew (Windows)Windows-1255
Thai (Windows)TIS-620
Turkish (Windows)Windows-1254
Vietnamese (Windows)Windows-1258
Western European (Windows)Windows-1252
ISO
Arabic (ISO)ISO-8859-6
Baltic (ISO)ISO-8859-4
Central European (ISO)ISO-8859-2
Cyrillic (ISO)ISO-8859-5
Estonian (ISO)ISO-8859-13
Greek (ISO)ISO-8859-7
Hebrew (ISO-Logical)ISO-8859-8-l
Hebrew (ISO-Visual)ISO-8859-8
Latin 9 (ISO)ISO-8859-15
Turkish (ISO)ISO-8859-9
Western European (ISO)ISO-8859-1
Other
Chinese Simplified (GB18030)GB18030
Chinese Simplified (GB2312)GB2312
Chinese Simplified (HZ)HZ
Chinese Traditional (Big5)Big5
Japanese (Shift-JIS)Shift_JIS
Japanese (EUC)EUC-JP
KoreanEUC-KR
Unicode (UTF-8)UTF-8
+ +

Internet Explorer does not recognize some of the more obscure +character encodings, and having to lookup the real names with a table +is a pain, so I recommend using Mozilla Firefox to find out your +character encoding.

+ +

Finding the embedded encoding

+ +

At this point, you may be asking, "Didn't we already find out our +encoding?" Well, as it turns out, there are multiple places where +a web developer can specify a character encoding, and one such place +is in a META tag:

+ +
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ +

You'll find this in the HEAD section of an HTML document. +The text to the right of charset= is the "claimed" +encoding: the HTML claims to be this encoding, but whether or not this +is actually the case depends on other factors. For now, take note +if your META tag claims that either:

+ +
    +
  1. The character encoding is the same as the one reported by the + browser,
    2. +
  2. The character encoding is different from the browser's, or
    3. +
  3. There is no META tag at all! (horror, horror!)
    4. +
+ +

Fixing the encoding

+ +

The advice given here is for pages being served as +vanilla text/html. Different practices must be used +for application/xml or application/xml+xhtml, see +W3C's +document on XHTML media types for more information.

+ +

If your META encoding and your real encoding match, +savvy! You can skip this section. If they don't...

+ +

No embedded encoding

+ +

If this is the case, you'll want to add in the appropriate +META tag to your website. It's as simple as copy-pasting +the code snippet above and replacing UTF-8 with whatever is the mime name +of your real encoding.

+ +
+

For all those skeptics out there, there is a very good reason + why the character encoding should be explicitly stated. When the + browser isn't told what the character encoding of a text is, it + has to guess: and sometimes the guess is wrong. Hackers can manipulate + this guess in order to slip XSS past filters and then fool the + browser into executing it as active code. A great example of this + is the Google UTF-7 + exploit.

+

You might be able to get away with not specifying a character + encoding with the META tag as long as your webserver + sends the right Content-Type header, but why risk it? Besides, if + the user downloads the HTML file, there is no longer any webserver + to define the character encoding.

+
+ +

Embedded encoding disagrees

+ +

This is an extremely common mistake: another source is telling +the browser what the +character encoding is and is overriding the embedded encoding. This +source usually is the Content-Type HTTP header that the webserver (i.e. +Apache) sends. A usual Content-Type header sent with a page might +look like this:

+ +
Content-Type: text/html; charset=ISO-8859-1
+ +

Notice how there is a charset parameter: this is the webserver's +way of telling a browser what the character encoding is, much like +the META tags we touched upon previously.

+ +

In fact, the META tag is +designed as a substitute for the HTTP header for contexts where +sending headers is impossible (such as locally stored files without +a webserver). Thus the name http-equiv (HTTP equivalent). +

+ +

There are two ways to go about fixing this: changing the META +tag to match the HTTP header, or changing the HTTP header to match +the META tag. How do we know which to do? It depends +on the website's content: after all, headers and tags are only ways of +describing the actual characters on the web page.

+ +

If your website:

+ +
+
...only uses ASCII characters,
+
Either way is fine, but I recommend switching both to + UTF-8 (more on this later).
+
...uses special characters, and they display + properly,
+
Change the embedded encoding to the server encoding.
+
...uses special characters, but users often complain that + they come out garbled,
+
Change the server encoding to the embedded encoding.
+
+ +

Changing a META tag is easy: just swap out the old encoding +for the new. Changing the server (HTTP header) encoding, however, +is slightly more difficult.

+ +

Changing the server encoding

+ +

PHP header() function

+ +

The simplest way to handle this problem is to send the encoding +yourself, via your programming language. Since you're using HTML +Purifier, I'll assume PHP, although it's not too difficult to do +similar things in +other +languages. The appropriate code is:

+ +
header('Content-Type:text/html; charset=UTF-8');
+ +

...replacing UTF-8 with whatever your embedded encoding is. +This code must come before any output, so be careful about +stray whitespace in your application (i.e., any whitespace before +output excluding whitespace within <?php ?> tags).

+ +

PHP ini directive

+ +

PHP also has a neat little ini directive that can save you a +header call: default_charset. Using this code:

+ +
ini_set('default_charset', 'UTF-8');
+ +

...will also do the trick. If PHP is running as an Apache module (and +not as FastCGI, consult +phpinfo() for details), you can even use htaccess to apply this property +across many PHP files:

+ +
php_value default_charset "UTF-8"
+ +

As with all INI directives, this can +also go in your php.ini file. Some hosting providers allow you to customize +your own php.ini file, ask your support for details. Use:

+
default_charset = "utf-8"
+ +

Non-PHP

+ +

You may, for whatever reason, need to set the character encoding +on non-PHP files, usually plain ol' HTML files. Doing this +is more of a hit-or-miss process: depending on the software being +used as a webserver and the configuration of that software, certain +techniques may work, or may not work.

+ +

.htaccess

+ +

On Apache, you can use an .htaccess file to change the character +encoding. I'll defer to +W3C +for the in-depth explanation, but it boils down to creating a file +named .htaccess with the contents:

+ +
AddCharset UTF-8 .html
+ +

Where UTF-8 is replaced with the character encoding you want to +use and .html is a file extension that this will be applied to. This +character encoding will then be set for any file directly in +or in the subdirectories of directory you place this file in.

+ +

If you're feeling particularly courageous, you can use:

+ +
AddDefaultCharset UTF-8
+ +

...which changes the character set Apache adds to any document that +doesn't have any Content-Type parameters. This directive, which the +default configuration file sets to iso-8859-1 for security +reasons, is probably why your headers mismatch +with the META tag. If you would prefer Apache not to be +butting in on your character encodings, you can tell it not +to send anything at all:

+ +
AddDefaultCharset Off
+ +

...making your internal charset declaration (usually the META tags) +the sole source of character encoding +information. In these cases, it is especially important to make +sure you have valid META tags on your pages and all the +text before them is ASCII.

+ +

These directives can also be +placed in httpd.conf file for Apache, but +in most shared hosting situations you won't be able to edit this file. +

+ +

File extensions

+ +

If you're not allowed to use .htaccess files, you can often +piggy-back off of Apache's default AddCharset declarations to get +your files in the proper extension. Here are Apache's default +character set declarations:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CharsetFile extension(s)
ISO-8859-1.iso8859-1 .latin1
ISO-8859-2.iso8859-2 .latin2 .cen
ISO-8859-3.iso8859-3 .latin3
ISO-8859-4.iso8859-4 .latin4
ISO-8859-5.iso8859-5 .latin5 .cyr .iso-ru
ISO-8859-6.iso8859-6 .latin6 .arb
ISO-8859-7.iso8859-7 .latin7 .grk
ISO-8859-8.iso8859-8 .latin8 .heb
ISO-8859-9.iso8859-9 .latin9 .trk
ISO-2022-JP.iso2022-jp .jis
ISO-2022-KR.iso2022-kr .kis
ISO-2022-CN.iso2022-cn .cis
Big5.Big5 .big5 .b5
WINDOWS-1251.cp-1251 .win-1251
CP866.cp866
KOI8-r.koi8-r .koi8-ru
KOI8-ru.koi8-uk .ua
ISO-10646-UCS-2.ucs2
ISO-10646-UCS-4.ucs4
UTF-8.utf8
GB2312.gb2312 .gb
utf-7.utf7
EUC-TW.euc-tw
EUC-JP.euc-jp
EUC-KR.euc-kr
shift_jis.sjis
+ +

So, for example, a file named page.utf8.html or +page.html.utf8 will probably be sent with the UTF-8 charset +attached, the difference being that if there is an +AddCharset charset .html declaration, it will override +the .utf8 extension in page.utf8.html (precedence moves +from right to left). By default, Apache has no such declaration.

+ +

Microsoft IIS

+ +

If anyone can contribute information on how to configure Microsoft +IIS to change character encodings, I'd be grateful.

+ +

XML

+ +

META tags are the most common source of embedded +encodings, but they can also come from somewhere else: XML +Declarations. They look like:

+ +
<?xml version="1.0" encoding="UTF-8"?>
+ +

...and are most often found in XML documents (including XHTML).

+ +

For XHTML, this XML Declaration theoretically +overrides the META tag. In reality, this happens only when the +XHTML is actually served as legit XML and not HTML, which is almost always +never due to Internet Explorer's lack of support for +application/xhtml+xml (even though doing so is often +argued to be good +practice and is required by the XHTML 1.1 specification).

+ +

For XML, however, this XML Declaration is extremely important. +Since most webservers are not configured to send charsets for .xml files, +this is the only thing a parser has to go on. Furthermore, the default +for XML files is UTF-8, which often butts heads with more common +ISO-8859-1 encoding (you see this in garbled RSS feeds).

+ +

In short, if you use XHTML and have gone through the +trouble of adding the XML Declaration, make sure it jives +with your META tags (which should only be present +if served in text/html) and HTTP headers.

+ +

Inside the process

+ +

This section is not required reading, +but may answer some of your questions on what's going on in all +this character encoding hocus pocus. If you're interested in +moving on to the next phase, skip this section.

+ +

A logical question that follows all of our wheeling and dealing +with multiple sources of character encodings is "Why are there +so many options?" To answer this question, we have to turn +back our definition of character encodings: they allow a program +to interpret bytes into human-readable characters.

+ +

Thus, a chicken-egg problem: a character encoding +is necessary to interpret the +text of a document. A META tag is in the text of a document. +The META tag gives the character encoding. How can we +determine the contents of a META tag, inside the text, +if we don't know it's character encoding? And how do we figure out +the character encoding, if we don't know the contents of the +META tag?

+ +

Fortunately for us, the characters we need to write the +META are in ASCII, which is pretty much universal +over every character encoding that is in common use today. So, +all the web-browser has to do is parse all the way down until +it gets to the Content-Type tag, extract the character encoding +tag, then re-parse the document according to this new information.

+ +

Obviously this is complicated, so browsers prefer the simpler +and more efficient solution: get the character encoding from a +somewhere other than the document itself, i.e. the HTTP headers, +much to the chagrin of HTML authors who can't set these headers.

+ +

Why UTF-8?

+ +

So, you've gone through all the trouble of ensuring that your +server and embedded characters all line up properly and are +present. Good job: at +this point, you could quit and rest easy knowing that your pages +are not vulnerable to character encoding style XSS attacks. +However, just as having a character encoding is better than +having no character encoding at all, having UTF-8 as your +character encoding is better than having some other random +character encoding, and the next step is to convert to UTF-8. +But why?

+ +

Internationalization

+ +

Many software projects, at one point or another, suddenly realize +that they should be supporting more than one language. Even regular +usage in one language sometimes requires the occasional special character +that, without surprise, is not available in your character set. Sometimes +developers get around this by adding support for multiple encodings: when +using Chinese, use Big5, when using Japanese, use Shift-JIS, when +using Greek, etc. Other times, they use character references with great +zeal.

+ +

UTF-8, however, obviates the need for any of these complicated +measures. After getting the system to use UTF-8 and adjusting for +sources that are outside the hand of the browser (more on this later), +UTF-8 just works. You can use it for any language, even many languages +at once, you don't have to worry about managing multiple encodings, +you don't have to use those user-unfriendly entities.

+ +

User-friendly

+ +

Websites encoded in Latin-1 (ISO-8859-1) which occasionally need +a special character outside of their scope often will use a character +entity reference to achieve the desired effect. For instance, θ can be +written &theta;, regardless of the character encoding's +support of Greek letters.

+ +

This works nicely for limited use of special characters, but +say you wanted this sentence of Chinese text: 激光, +這兩個字是甚麼意思. +The ampersand encoded version would look like this:

+ +
&#28608;&#20809;, &#36889;&#20841;&#20491;&#23383;&#26159;&#29978;&#40636;&#24847;&#24605;
+ +

Extremely inconvenient for those of us who actually know what +character entities are, totally unintelligible to poor users who don't! +Even the slightly more user-friendly, "intelligible" character +entities like &theta; will leave users who are +uninterested in learning HTML scratching their heads. On the other +hand, if they see θ in an edit box, they'll know that it's a +special character, and treat it accordingly, even if they don't know +how to write that character themselves.

+ +

Wikipedia is a great case study for +an application that originally used ISO-8859-1 but switched to UTF-8 +when it became far to cumbersome to support foreign languages. Bots +will now actually go through articles and convert character entities +to their corresponding real characters for the sake of user-friendliness +and searchability. See +Meta's +page on special characters for more details. +

+ +

Forms

+ +

While we're on the tack of users, how do non-UTF-8 web forms deal +with characters that are outside of their character set? Rather than +discuss what UTF-8 does right, we're going to show what could go wrong +if you didn't use UTF-8 and people tried to use characters outside +of your character encoding.

+ +

The troubles are large, extensive, and extremely difficult to fix (or, +at least, difficult enough that if you had the time and resources to invest +in doing the fix, you would be probably better off migrating to UTF-8). +There are two types of form submission: application/x-www-form-urlencoded +which is used for GET and by default for POST, and multipart/form-data +which may be used by POST, and is required when you want to upload +files.

+ +

The following is a summarization of notes from + +FORM submission and i18n. That document contains lots +of useful information, but is written in a rambly manner, so +here I try to get right to the point. (Note: the original has +disappeared off the web, so I am linking to the Web Archive copy.)

+ +

application/x-www-form-urlencoded

+ +

This is the Content-Type that GET requests must use, and POST requests +use by default. It involves the ubiquitous percent encoding format that +looks something like: %C3%86. There is no official way of +determining the character encoding of such a request, since the percent +encoding operates on a byte level, so it is usually assumed that it +is the same as the encoding the page containing the form was submitted +in. (RFC 3986 +recommends that textual identifiers be translated to UTF-8; however, browser +compliance is spotty.) You'll run into very few problems +if you only use characters in the character encoding you chose.

+ +

However, once you start adding characters outside of your encoding +(and this is a lot more common than you may think: take curly +"smart" quotes from Microsoft as an example), +a whole manner of strange things start to happen. Depending on the +browser you're using, they might:

+ +
    +
  • Replace the unsupported characters with useless question marks,
    • +
  • Attempt to fix the characters (example: smart quotes to regular quotes),
    • +
  • Replace the character with a character entity reference, or
    • +
  • Send it anyway as a different character encoding mixed in + with the original encoding (usually Windows-1252 rather than + iso-8859-1 or UTF-8 interspersed in 8-bit)
    • +
+ +

To properly guard against these behaviors, you'd have to sniff out +the browser agent, compile a database of different behaviors, and +take appropriate conversion action against the string (disregarding +a spate of extremely mysterious, random and devastating bugs Internet +Explorer manifests every once in a while). Or you could +use UTF-8 and rest easy knowing that none of this could possibly happen +since UTF-8 supports every character.

+ +

multipart/form-data

+ +

Multipart form submission takes away a lot of the ambiguity +that percent-encoding had: the server now can explicitly ask for +certain encodings, and the client can explicitly tell the server +during the form submission what encoding the fields are in.

+ +

There are two ways you go with this functionality: leave it +unset and have the browser send in the same encoding as the page, +or set it to UTF-8 and then do another conversion server-side. +Each method has deficiencies, especially the former.

+ +

If you tell the browser to send the form in the same encoding as +the page, you still have the trouble of what to do with characters +that are outside of the character encoding's range. The behavior, once +again, varies: Firefox 2.0 converts them to character entity references +while Internet Explorer 7.0 mangles them beyond intelligibility. For +serious internationalization purposes, this is not an option.

+ +

The other possibility is to set Accept-Encoding to UTF-8, which +begs the question: Why aren't you using UTF-8 for everything then? +This route is more palatable, but there's a notable caveat: your data +will come in as UTF-8, so you will have to explicitly convert it into +your favored local character encoding.

+ +

I object to this approach on idealogical grounds: you're +digging yourself deeper into +the hole when you could have been converting to UTF-8 +instead. And, of course, you can't use this method for GET requests.

+ +

Well supported

+ +

Almost every modern browser in the wild today has full UTF-8 and Unicode +support: the number of troublesome cases can be counted with the +fingers of one hand, and these browsers usually have trouble with +other character encodings too. Problems users usually encounter stem +from the lack of appropriate fonts to display the characters (once +again, this applies to all character encodings and HTML entities) or +Internet Explorer's lack of intelligent font picking (which can be +worked around).

+ +

We will go into more detail about how to deal with edge cases in +the browser world in the Migration section, but rest assured that +converting to UTF-8, if done correctly, will not result in users +hounding you about broken pages.

+ +

HTML Purifier

+ +

And finally, we get to HTML Purifier. HTML Purifier is built to +deal with UTF-8: any indications otherwise are the result of an +encoder that converts text from your preferred encoding to UTF-8, and +back again. HTML Purifier never touches anything else, and leaves +it up to the module iconv to do the dirty work.

+ +

This approach, however, is not perfect. iconv is blithely unaware +of HTML character entities. HTML Purifier, in order to +protect against sophisticated escaping schemes, normalizes all character +and numeric entity references before processing the text. This leads to +one important ramification:

+ +

Any character that is not supported by the target character +set, regardless of whether or not it is in the form of a character +entity reference or a raw character, will be silently ignored.

+ +

Example of this principle at work: say you have &theta; +in your HTML, but the output is in Latin-1 (which, understandably, +does not understand Greek), the following process will occur (assuming you've +set the encoding correctly using %Core.Encoding):

+ +
    +
  • The Encoder will transform the text from ISO 8859-1 to UTF-8 + (note that theta is preserved here since it doesn't actually use + any non-ASCII characters): &theta;
    • +
  • The EntityParser will transform all named and numeric + character entities to their corresponding raw UTF-8 equivalents: + θ
    • +
  • HTML Purifier processes the code: θ
    • +
  • The Encoder now transforms the text back from UTF-8 + to ISO 8859-1. Since Greek is not supported by ISO 8859-1, it + will be either ignored or replaced with a question mark: + ?
    • +
+ +

This behaviour is quite unsatisfactory. It is a deal-breaker for +international applications, and it can be mildly annoying for the provincial +soul who occasionally needs a special character. Since 1.4.0, HTML +Purifier has provided a slightly more palatable workaround using +%Core.EscapeNonASCIICharacters. The process now looks like:

+ +
    +
  • The Encoder transforms encoding to UTF-8: &theta;
    • +
  • The EntityParser transforms entities: θ
    • +
  • HTML Purifier processes the code: θ
    • +
  • The Encoder replaces all non-ASCII characters + with numeric entity reference: &#952;
    • +
  • For good measure, Encoder transforms encoding back to + original (which is strictly unnecessary for 99% of encodings + out there): &#952; (remember, it's all ASCII!)
    • +
+ +

...which means that this is only good for an occasional foray into +the land of Unicode characters, and is totally unacceptable for Chinese +or Japanese texts. The even bigger kicker is that, supposing the +input encoding was actually ISO-8859-7, which does support +theta, the character would get converted into a character entity reference +anyway! (The Encoder does not discriminate).

+ +

The current functionality is about where HTML Purifier will be for +the rest of eternity. HTML Purifier could attempt to preserve the original +form of the character references so that they could be substituted back in, only the +DOM extension kills them off irreversibly. HTML Purifier could also attempt +to be smart and only convert non-ASCII characters that weren't supported +by the target encoding, but that would require reimplementing iconv +with HTML awareness, something I will not do.

+ +

So there: either it's UTF-8 or crippled international support. Your pick! (and I'm +not being sarcastic here: some people could care less about other languages).

+ +

Migrate to UTF-8

+ +

So, you've decided to bite the bullet, and want to migrate to UTF-8. +Note that this is not for the faint-hearted, and you should expect +the process to take longer than you think it will take.

+ +

The general idea is that you convert all existing text to UTF-8, +and then you set all the headers and META tags we discussed earlier +to UTF-8. There are many ways going about doing this: you could +write a conversion script that runs through the database and re-encodes +everything as UTF-8 or you could do the conversion on the fly when someone +reads the page. The details depend on your system, but I will cover +some of the more subtle points of migration that may trip you up.

+ +

Configuring your database

+ +

Most modern databases, the most prominent open-source ones being MySQL +4.1+ and PostgreSQL, support character encodings. If you're switching +to UTF-8, logically speaking, you'd want to make sure your database +knows about the change too. There are some caveats though:

+ +

Legit method

+ +

Standardization in terms of SQL syntax for specifying character +encodings is notoriously spotty. Refer to your respective database's +documentation on how to do this properly.

+ +

For MySQL, ALTER will magically perform the +character encoding conversion for you. However, you have +to make sure that the text inside the column is what is says it is: +if you had put Shift-JIS in an ISO 8859-1 column, MySQL will irreversibly mangle +the text when you try to convert it to UTF-8. You'll have to convert +it to a binary field, convert it to a Shift-JIS field (the real encoding), +and then finally to UTF-8. Many a website had pages irreversibly mangled +because they didn't realize that they'd been deluding themselves about +the character encoding all along; don't become the next victim.

+ +

For PostgreSQL, there appears to be no direct way to change the +encoding of a database (as of 8.2). You will have to dump the data, and then reimport +it into a new table. Make sure that your client encoding is set properly: +this is how PostgreSQL knows to perform an encoding conversion.

+ +

Many times, you will be also asked about the "collation" of +the new column. Collation is how a DBMS sorts text, like ordering +B, C and A into A, B and C (the problem gets surprisingly complicated +when you get to languages like Thai and Japanese). If in doubt, +going with the default setting is usually a safe bet.

+ +

Once the conversion is all said and done, you still have to remember +to set the client encoding (your encoding) properly on each database +connection using SET NAMES (which is standard SQL and is +usually supported).

+ +

Binary

+ +

Due to the aforementioned compatibility issues, a more interoperable +way of storing UTF-8 text is to stuff it in a binary datatype. +CHAR becomes BINARY, VARCHAR becomes +VARBINARY and TEXT becomes BLOB. +Doing so can save you some huge headaches:

+ +
    +
  • The syntax for binary data types is very portable,
    • +
  • MySQL 4.0 has no support for character encodings, so + if you want to support it you have to use binary,
    • +
  • MySQL, as of 5.1, has no support for four byte UTF-8 characters, + which represent characters beyond the basic multilingual + plane, and
    • +
  • You will never have to worry about your DBMS being too smart + and attempting to convert your text when you don't want it to.
    • +
+ +

MediaWiki, a very prominent international application, uses binary fields +for storing their data because of point three.

+ +

There are drawbacks, of course:

+ +
    +
  • Database tools like PHPMyAdmin won't be able to offer you inline + text editing, since it is declared as binary,
    • +
  • It's not semantically correct: it's really text not binary + (lying to the database),
    • +
  • Unless you use the not-very-portable wizardry mentioned above, + you have to change the encoding yourself (usually, you'd do + it on the fly), and
    • +
  • You will not have collation.
    • +
+ +

Choose based on your circumstances.

+ +

Text editor

+ +

For more flat-file oriented systems, you will often be tasked with +converting reams of existing text and HTML files into UTF-8, as well as +making sure that all new files uploaded are properly encoded. Once again, +I can only point vaguely in the right direction for converting your +existing files: make sure you backup, make sure you use +iconv(), and +make sure you know what the original character encoding of the files +is (or are, depending on the tidiness of your system).

+ +

However, I can proffer more specific advice on the subject of +text editors. Many text editors have notoriously spotty Unicode support. +To find out how your editor is doing, you can check out this list +or Wikipedia's list. +I personally use Notepad++, which works like a charm when it comes to UTF-8. +Usually, you will have to explicitly tell the editor through some dialogue +(usually Save as or Format) what encoding you want it to use. An editor +will often offer "Unicode" as a method of saving, which is +ambiguous. Make sure you know whether or not they really mean UTF-8 +or UTF-16 (which is another flavor of Unicode).

+ +

The two things to look out for are whether or not the editor +supports font mixing (multiple +fonts in one document) and whether or not it adds a BOM. +Font mixing is important because fonts rarely have support for every +language known to mankind: in order to be flexible, an editor must +be able to take a little from here and a little from there, otherwise +all your Chinese characters will come as nice boxes. We'll discuss +BOM below.

+ +

Byte Order Mark (headers already sent!)

+ +

The BOM, or Byte +Order Mark, is a magical, invisible character placed at +the beginning of UTF-8 files to tell people what the encoding is and +what the endianness of the text is. It is also unnecessary.

+ +

Because it's invisible, it often +catches people by surprise when it starts doing things it shouldn't +be doing. For example, this PHP file:

+ +
BOM<?php
+header('Location: index.php');
+?>
+ +

...will fail with the all too familiar Headers already sent +PHP error. And because the BOM is invisible, this culprit will go unnoticed. +My suggestion is to only use ASCII in PHP pages, but if you must, make +sure the page is saved WITHOUT the BOM.

+ +
+

The headers the error is referring to are HTTP headers, + which are sent to the browser before any HTML to tell it various + information. The moment any regular text (and yes, a BOM counts as + ordinary text) is output, the headers must be sent, and you are + not allowed to send anymore. Thus, the error.

+
+ +

If you are reading in text files to insert into the middle of another +page, it is strongly advised (but not strictly necessary) that you replace out the UTF-8 byte +sequence for BOM "\xEF\xBB\xBF" before inserting it in, +via:

+ +
$text = str_replace("\xEF\xBB\xBF", '', $text);
+ +

Fonts

+ +

Generally speaking, people who are having trouble with fonts fall +into two categories:

+ +
    +
  • Those who want to +use an extremely obscure language for which there is very little +support even among native speakers of the language, and
    • +
  • Those where the primary language of the text is +well-supported but there are occasional characters +that aren't supported.
    • +
+ +

Yes, there's always a chance where an English user happens across +a Sinhalese website and doesn't have the right font. But an English user +who happens not to have the right fonts probably has no business reading Sinhalese +anyway. So we'll deal with the other two edge cases.

+ +

Obscure scripts

+ +

If you run a Bengali website, you may get comments from users who +would like to read your website but get heaps of question marks or +other meaningless characters. Fixing this problem requires the +installation of a font or language pack which is often highly +dependent on what the language is. Here is an example +of such a help file for the Bengali language; I am sure there are +others out there too. You just have to point users to the appropriate +help file.

+ +

Occasional use

+ +

A prime example of when you'll see some very obscure Unicode +characters embedded in what otherwise would be very bland ASCII are +letters of the +International +Phonetic Alphabet (IPA), use to designate pronunciations in a very standard +manner (you probably see them all the time in your dictionary). Your +average font probably won't have support for all of the IPA characters +like ʘ (bilabial click) or ʒ (voiced postalveolar fricative). +So what's a poor browser to do? Font mix! Smart browsers like Mozilla Firefox +and Internet Explorer 7 will borrow glyphs from other fonts in order +to make sure that all the characters display properly.

+ +

But what happens when the browser isn't smart and happens to be the +most widely used browser in the entire world? Microsoft IE 6 +is not smart enough to borrow from other fonts when a character isn't +present, so more often than not you'll be slapped with a nice big �. +To get things to work, MSIE 6 needs a little nudge. You could configure it +to use a different font to render the text, but you can achieve the same +effect by selectively changing the font for blocks of special characters +to known good Unicode fonts.

+ +

Fortunately, the folks over at Wikipedia have already done all the +heavy lifting for you. Get the CSS from the horses mouth here: +Common.css, +and search for ".IPA" There are also a smattering of +other classes you can use for other purposes, check out +this page +for more details. For you lazy ones, this should work:

+ +
.Unicode {
+        font-family: Code2000, "TITUS Cyberbit Basic", "Doulos SIL",
+            "Chrysanthi Unicode", "Bitstream Cyberbit",
+            "Bitstream CyberBase", Thryomanes, Gentium, GentiumAlt,
+            "Lucida Grande", "Arial Unicode MS", "Microsoft Sans Serif",
+            "Lucida Sans Unicode";
+        font-family /**/:inherit; /* resets fonts for everyone but IE6 */
+}
+ +

The standard usage goes along the lines of <span class="Unicode">Crazy +Unicode stuff here</span>. Characters in the +Windows Glyph List +usually don't need to be fixed, but for anything else you probably +want to play it safe. Unless, of course, you don't care about IE6 +users.

+ +

Dealing with variable width in functions

+ +

When people claim that PHP6 will solve all our Unicode problems, they're +misinformed. It will not fix any of the aforementioned troubles. It will, +however, fix the problem we are about to discuss: processing UTF-8 text +in PHP.

+ +

PHP (as of PHP5) is blithely unaware of the existence of UTF-8 (with a few +notable exceptions). Sometimes, this will cause problems, other times, +this won't. So far, we've avoided discussing the architecture of +UTF-8, so, we must first ask, what is UTF-8? Yes, it supports Unicode, +and yes, it is variable width. Other traits:

+ +
    +
  • Every character's byte sequence is unique and will never be found + inside the byte sequence of another character,
    • +
  • UTF-8 may use up to four bytes to encode a character,
    • +
  • UTF-8 text must be checked for well-formedness,
    • +
  • Pure ASCII is also valid UTF-8, and
    • +
  • Binary sorting will sort UTF-8 in the same order as Unicode.
    • +
+ +

Each of these traits affect different domains of text processing +in different ways. It is beyond the scope of this document to explain +what precisely these implications are. PHPWact provides +a very good reference document +on what to expect from each function, although coverage is spotty in +some areas. Their more general notes on +character sets +are also worth looking at for information on UTF-8. Some rules of thumb +when dealing with Unicode text:

+ +
    +
  • Do not EVER use functions that:
      +
    • ...convert case (strtolower, strtoupper, ucfirst, ucwords)
      • +
    • ...claim to be case-insensitive (str_ireplace, stristr, strcasecmp)
      • +
    • +
  • Think twice before using functions that:
      +
    • ...count characters (strlen will return bytes, not characters; + str_split and word_wrap may corrupt)
      • +
    • ...convert characters to entity references (UTF-8 doesn't need entities)
      • +
    • ...do very complex string processing (*printf)
      • +
    • +
+ +

Note: this list applies to UTF-8 encoded text only: if you have +a string that you are 100% sure is ASCII, be my guest and use +strtolower (HTML Purifier uses this function.)

+ +

Regardless, always think in bytes, not characters. If you use strpos() +to find the position of a character, it will be in bytes, but this +usually won't matter since substr() also operates with byte indices!

+ +

You'll also need to make sure your UTF-8 is well-formed and will +probably need replacements for some of these functions. I recommend +using Harry Fuecks' PHP +UTF-8 library, rather than use mb_string directly. HTML Purifier +also defines a few useful UTF-8 compatible functions: check out +Encoder.php in the /library/HTMLPurifier/ +directory.

+ + + +

Well, that's it. Hopefully this document has served as a very +practical springboard into knowledge of how UTF-8 works. You may have +decided that you don't want to migrate yet: that's fine, just know +what will happen to your output and what bug reports you may receive.

+ +

Many other developers have already discussed the subject of Unicode, +UTF-8 and internationalization, and I would like to defer to them for +a more in-depth look into character sets and encodings.

+ + + + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/enduser-youtube.html b/vendor/ezyang/htmlpurifier/docs/enduser-youtube.html new file mode 100644 index 0000000000..87a36b9aaa --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/enduser-youtube.html @@ -0,0 +1,153 @@ + + + + + + + +Embedding YouTube Videos - HTML Purifier + + + +

Embedding YouTube Videos

+
...as well as other dangerous active content
+ +
Filed under End-User
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

Clients like their YouTube videos. It gives them a warm fuzzy feeling when +they see a neat little embedded video player on their websites that can play +the latest clips from their documentary "Fido and the Bones of Spring". +All joking aside, the ability to embed YouTube videos or other active +content in their pages is something that a lot of people like.

+ +

This is a bad idea. The moment you embed anything untrusted, +you will definitely be slammed by a manner of nasties that can be +embedded in things from your run of the mill Flash movie to +Quicktime movies. +Even img tags, which HTML Purifier allows by default, can be +dangerous. Be distrustful of anything that tells a browser to load content +from another website automatically.

+ +

Luckily for us, however, whitelisting saves the day. Sure, letting users +include any old random flash file could be dangerous, but if it's +from a specific website, it probably is okay. If no amount of pleading will +convince the people upstairs that they should just settle with just linking +to their movies, you may find this technique very useful.

+ +

Looking in

+ +

Below is custom code that allows users to embed +YouTube videos. This is not favoritism: this trick can easily be adapted for +other forms of embeddable content.

+ +

Usually, websites like YouTube give us boilerplate code that you can insert +into your documents. YouTube's code goes like this:

+ +
+<object width="425" height="350">
+  <param name="movie" value="http://www.youtube.com/v/AyPzM5WK8ys" />
+  <param name="wmode" value="transparent" />
+  <embed src="http://www.youtube.com/v/AyPzM5WK8ys"
+         type="application/x-shockwave-flash"
+         wmode="transparent" width="425" height="350" />
+</object>
+
+ +

There are two things to note about this code:

+ +
    +
  1. <embed> is not recognized by W3C, so if you want + standards-compliant code, you'll have to get rid of it.
    2. +
  2. The code is exactly the same for all instances, except for the + identifier AyPzM5WK8ys which tells us which movie file + to retrieve.
    3. +
+ +

What point 2 means is that if we have code like <span +class="youtube-embed">AyPzM5WK8ys</span> your +application can reconstruct the full object from this small snippet that +passes through HTML Purifier unharmed. +Show me the code!

+ +

And the corresponding usage:

+ +
<?php
+    $config->set('Filter.YouTube', true);
+?>
+ +

There is a bit going in the two code snippets, so let's explain.

+ +
    +
  1. This is a Filter object, which intercepts the HTML that is + coming into and out of the purifier. You can add as many + filter objects as you like. preFilter() + processes the code before it gets purified, and postFilter() + processes the code afterwards. So, we'll use preFilter() to + replace the object tag with a span, and postFilter() + to restore it.
    2. +
  2. The first preg_replace call replaces any YouTube code users may have + embedded into the benign span tag. Span is used because it is inline, + and objects are inline too. We are very careful to be extremely + restrictive on what goes inside the span tag, as if an errant code + gets in there it could get messy.
    3. +
  3. The HTML is then purified as usual.
    4. +
  4. Then, another preg_replace replaces the span tag with a fully fledged + object. Note that the embed is removed, and, in its place, a data + attribute was added to the object. This makes the tag standards + compliant! It also breaks Internet Explorer, so we add in a bit of + conditional comments with the old embed code to make it work again. + It's all quite convoluted but works.
    5. +
+ +

Warning

+ +

There are a number of possible problems with the code above, depending +on how you look at it.

+ +

Cannot change width and height

+ +

The width and height of the final YouTube movie cannot be adjusted. This +is because I am lazy. If you really insist on letting users change the size +of the movie, what you need to do is package up the attributes inside the +span tag (along with the movie ID). It gets complicated though: a malicious +user can specify an outrageously large height and width and attempt to crash +the user's operating system/browser. You need to either cap it by limiting +the amount of digits allowed in the regex or using a callback to check the +number.

+ +

Trusts media's host's security

+ +

By allowing this code onto our website, we are trusting that YouTube has +tech-savvy enough people not to allow their users to inject malicious +code into the Flash files. An exploit on YouTube means an exploit on your +site. Even though YouTube is run by the reputable Google, it +doesn't +mean they are +invulnerable. +You're putting a certain measure of the job on an external provider (just as +you have by entrusting your user input to HTML Purifier), and +it is important that you are cognizant of the risk.

+ +

Poorly written adaptations compromise security

+ +

This should go without saying, but if you're going to adapt this code +for Google Video or the like, make sure you do it right. It's +extremely easy to allow a character too many in postFilter() and +suddenly you're introducing XSS into HTML Purifier's XSS free output. HTML +Purifier may be well written, but it cannot guard against vulnerabilities +introduced after it has finished.

+ +

Help out!

+ +

If you write a filter for your favorite video destination (or anything +like that, for that matter), send it over and it might get included +with the core!

+ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/entities/xhtml-lat1.ent b/vendor/ezyang/htmlpurifier/docs/entities/xhtml-lat1.ent new file mode 100644 index 0000000000..ffee223eb1 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/entities/xhtml-lat1.ent @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/entities/xhtml-special.ent b/vendor/ezyang/htmlpurifier/docs/entities/xhtml-special.ent new file mode 100644 index 0000000000..ca358b2fec --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/entities/xhtml-special.ent @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/entities/xhtml-symbol.ent b/vendor/ezyang/htmlpurifier/docs/entities/xhtml-symbol.ent new file mode 100644 index 0000000000..63c2abfa6f --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/entities/xhtml-symbol.ent @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/examples/basic.php b/vendor/ezyang/htmlpurifier/docs/examples/basic.php new file mode 100644 index 0000000000..b51096d2d9 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/examples/basic.php @@ -0,0 +1,23 @@ +set('Core.Encoding', 'UTF-8'); // replace with your encoding +$config->set('HTML.Doctype', 'XHTML 1.0 Transitional'); // replace with your doctype + +$purifier = new HTMLPurifier($config); + +// untrusted input HTML +$html = 'Simple and short'; + +$pure_html = $purifier->purify($html); + +echo '
' . htmlspecialchars($pure_html) . '
'; + +// vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/fixquotes.htc b/vendor/ezyang/htmlpurifier/docs/fixquotes.htc new file mode 100644 index 0000000000..80dda2dc24 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/fixquotes.htc @@ -0,0 +1,9 @@ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/index.html b/vendor/ezyang/htmlpurifier/docs/index.html new file mode 100644 index 0000000000..3c4ecc7161 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/index.html @@ -0,0 +1,188 @@ + + + + + + + +Documentation - HTML Purifier + + + + +

Documentation

+ +

HTML Purifier has documentation for all types of people. +Here is an index of all of them.

+ +

End-user

+

End-user documentation that contains articles, tutorials and useful +information for casual developers using HTML Purifier.

+ +
+ +
IDs
+
Explains various methods for allowing IDs in documents safely.
+ +
Embedding YouTube videos
+
Explains how to safely allow the embedding of flash from trusted sites.
+ +
Speeding up HTML Purifier
+
Explains how to speed up HTML Purifier through caching or inbound filtering.
+ +
UTF-8: The Secret of Character Encoding
+
Describes the rationale for using UTF-8, the ramifications otherwise, and how to make the switch.
+ +
Tidy
+
Tutorial for tweaking HTML Purifier's Tidy-like behavior.
+ +
Customize
+
Tutorial for customizing HTML Purifier's tag and attribute sets.
+ +
URI Filters
+
Tutorial for creating custom URI filters.
+ +
+ +

Development

+

Developer documentation detailing code issues, roadmaps and project +conventions.

+ +
+ +
Implementation Progress
+
Tables detailing HTML element and CSS property implementation coverage.
+ +
Naming Conventions
+
Defines class naming conventions.
+ +
Optimization
+
Discusses possible methods of optimizing HTML Purifier.
+ +
Flushing the Purifier
+
Discusses when to flush HTML Purifier's various caches.
+ +
Advanced API
+
Specification for HTML Purifier's advanced API for defining +custom filtering behavior.
+ +
Config Schema
+
Describes config schema framework in HTML Purifier.
+ +
+ +

Proposals

+

Proposed features, as well as the associated rambling to get a clear +objective in place before attempted implementation.

+ +
+
Colors
+
Proposal to allow for color constraints.
+
+ +

Reference

+

Miscellaneous essays, research pieces and other reference type material +that may not directly discuss HTML Purifier.

+ +
+
DevNetwork Credits
+
Credits and links to DevNetwork forum topics.
+
+ +

Internal memos

+ +

Plaintext documents that are more for use by active developers of +the code. They may be upgraded to HTML files or stay as TXT scratchpads.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeNameDescription
End-userOverviewHigh level overview of the general control flow (mostly obsolete).
End-userSecurityCommon security issues that may still arise (half-baked).
DevelopmentConfig BC BreaksBackwards-incompatible changes in HTML Purifier 4.0.0
DevelopmentCode Quality IssuesEnumerates code quality issues and places that need to be refactored.
ProposalFilter levelsOutlines details of projected configurable level of filtering.
ProposalLanguageSpecification of I18N for error messages derived from MediaWiki (half-baked).
ProposalNew directivesAssorted configuration options that could be implemented.
ProposalCSS extractionTaking the inline CSS out of documents and into style.
ReferenceHandling Content Model ChangesDiscusses how to tidy up content model changes using custom ChildDef classes.
ReferenceProprietary tagsList of vendor-specific tags we may want to transform to W3C compliant markup.
ReferenceModularization of HTMLDefinitionProvides a high-level overview of the concepts behind HTMLModules.
ReferenceWHATWGHow WHATWG plays into what we need to do.
+ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/proposal-colors.html b/vendor/ezyang/htmlpurifier/docs/proposal-colors.html new file mode 100644 index 0000000000..6576338829 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/proposal-colors.html @@ -0,0 +1,49 @@ + + + + + + + +Proposal: Colors - HTML Purifier + + + +

Colors

+
Hammering some sense into those color-blind newbies
+ +
Filed under Proposals
+
Return to the index.
+
HTML Purifier End-User Documentation
+ +

Your website probably has a color-scheme. +Green on white, +purple on yellow, +whatever. When you give users the ability to style their content, you may +want them to keep in line with your styling. If you're website is all +about light colors, you don't want a user to come in and vandalize your +page with a deep maroon.

+ +

This is an extremely silly feature proposal, but I'm writing it down anyway.

+ +

What if the user could constrain the colors specified in inline styles? You +are only allowed to use these shades of dark green for text and these shades +of light yellow for the background. At the very least, you could ensure +that we did not have pale yellow on white text.

+ +

Implementation issues

+ +
    +
  1. Requires the color attribute definition to know, currently, what the text +and background colors are. This becomes difficult when classes are thrown +into the mix.
    2. +
  2. The user still has to define the permissible colors, how does one do +something like that?
    3. +
+ + + + + diff --git a/vendor/ezyang/htmlpurifier/docs/proposal-config.txt b/vendor/ezyang/htmlpurifier/docs/proposal-config.txt new file mode 100644 index 0000000000..4e031c5866 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/proposal-config.txt @@ -0,0 +1,23 @@ + +Configuration + +Configuration is documented on a per-use case: if a class uses a certain +value from the configuration object, it has to define its name and what the +value is used for. This means decentralized configuration declarations that +are nevertheless error checking and a centralized configuration object. + +Directives are divided into namespaces, indicating the major portion of +functionality they cover (although there may be overlaps). Please consult +the documentation in ConfigDef for more information on these namespaces. + +Since configuration is dependant on context, internal classes require a +configuration object to be passed as a parameter. (They also require a +Context object). A majority of classes do not need the config object, +but for those who do, it is a lifesaver. + +Definition objects are complex datatypes influenced by their respective +directive namespaces (HTMLDefinition with HTML and CSSDefinition with CSS). +If any of these directives is updated, HTML Purifier forces the definition +to be regenerated. + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/proposal-css-extraction.txt b/vendor/ezyang/htmlpurifier/docs/proposal-css-extraction.txt new file mode 100644 index 0000000000..9933c96b8f --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/proposal-css-extraction.txt @@ -0,0 +1,34 @@ + +Extracting inline CSS from HTML Purifier + voodoofied: Assigning semantics to elements + +Sander Tekelenburg brought to my attention the poor programming style of +inline CSS in HTML documents. In an ideal world, we wouldn't be using inline +CSS at all: everything would be assigned using semantic class attributes +from an external stylesheet. + +With ExtractStyleBlocks and CSSTidy, this is now possible (when allowed, users +can specify a style element which gets extracted from the user-submitted HTML, which +the application can place in the head of the HTML document). But there still +is the issue of inline CSS that refuses to go away. + +The basic idea behind this feature is assign every element a unique identifier, +and then move all of the CSS data to a style-sheet. This HTML: + +
Big things!
+ +into + +
Big things!
+ +and a stylesheet that is: + +#hp-12345 {text-align:center;} +#hp-12346 {color:red;} + +Beyond that, HTML Purifier can magically merge common CSS values together, +and a whole manner of other heuristic things. HTML Purifier should also +make it easy for an admin to re-style the HTML semantically. Speed is not +an issue. Also, better WYSIWYG editors are needed. + + vim: et sw=4 sts=4 diff --git a/vendor/ezyang/htmlpurifier/docs/proposal-errors.txt b/vendor/ezyang/htmlpurifier/docs/proposal-errors.txt new file mode 100644 index 0000000000..87cb2ac19e --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/proposal-errors.txt @@ -0,0 +1,211 @@ +Considerations for ErrorCollection + +Presently, HTML Purifier takes a code-execution centric approach to handling +errors. Errors are organized and grouped according to which segment of the +code triggers them, not necessarily the portion of the input document that +triggered the error. This means that errors are pseudo-sorted by category, +rather than location in the document. + +One easy way to "fix" this problem would be to re-sort according to line number. +However, the "category" style information we derive from naively following +program execution is still useful. After all, each of the strategies which +can report errors still process the document mostly linearly. Furthermore, +not only do they process linearly, but the way they pass off operations to +sub-systems mirrors that of the document. For example, AttrValidator will +linearly proceed through elements, and on each element will use AttrDef to +validate those contents. From there, the attribute might have more +sub-components, which have execution passed off accordingly. + +In fact, each strategy handles a very specific class of "error." + +RemoveForeignElements - element tokens +MakeWellFormed - element token ordering +FixNesting - element token ordering +ValidateAttributes - attributes of elements + +The crucial point is that while we care about the hierarchy governing these +different errors, we *don't* care about any other information about what actually +happens to the elements. This brings up another point: if HTML Purifier fixes +something, this is not really a notice/warning/error; it's really a suggestion +of a way to fix the aforementioned defects. + +In short, the refactoring to take this into account kinda sucks. + +Errors should not be recorded in order that they are reported. Instead, they +should be bound to the line (and preferably element) in which they were found. +This means we need some way to uniquely identify every element in the document, +which doesn't presently exist. An easy way of adding this would be to track +line columns. An important ramification of this is that we *must* use the +DirectLex implementation. + + 1. Implement column numbers for DirectLex [DONE!] + 2. Disable error collection when not using DirectLex [DONE!] + +Next, we need to re-orient all of the error declarations to place CurrentToken +at utmost important. Since this is passed via Context, it's not always clear +if that's available. ErrorCollector should complain HARD if it isn't available. +There are some locations when we don't have a token available. These include: + + * Lexing - this can actually have a row and column, but NOT correspond to + a token + * End of document errors - bump this to the end + +Actually, we *don't* have to complain if CurrentToken isn't available; we just +set it as a document-wide error. And actually, nothing needs to be done here. + +Something interesting to consider is whether or not we care about the locations +of attributes and CSS properties, i.e. the sub-objects that compose these things. +In terms of consistency, at the very least attributes should have column/line +numbers attached to them. However, this may be overkill, as attributes are +uniquely identifiable. You could go even further, with CSS, but they are also +uniquely identifiable. + +Bottom-line is, however, this information must be available, in form of the +CurrentAttribute and CurrentCssProperty (theoretical) context variables, and +it must be used to organize the errors that the sub-processes may throw. +There is also a hierarchy of sorts that may make merging this into one context +variable more sense, if it hadn't been for HTML's reasonably rigid structure. +A CSS property will never contain an HTML attribute. So we won't ever get +recursive relations, and having multiple depths won't ever make sense. Leave +this be. + +We already have this information, and consequently, using start and end is +*unnecessary*, so long as the context variables are set appropriately. We don't +care if an error was thrown by an attribute transform or an attribute definition; +to the end user these are the same (for a developer, they are different, but +they're better off with a stack trace (which we should add support for) in such +cases). + + 3. Remove start()/end() code. Don't get rid of recursion, though [DONE] + 4. Setup ErrorCollector to use context information to setup hierarchies. + This may require a different internal format. Use objects if it gets + complex. [DONE] + + ASIDE + More on this topic: since we are now binding errors to lines + and columns, a particular error can have three relationships to that + specific location: + + 1. The token at that location directly + RemoveForeignElements + AttrValidator (transforms) + MakeWellFormed + 2. A "component" of that token (i.e. attribute) + AttrValidator (removals) + 3. A modification to that node (i.e. contents from start to end + token) as a whole + FixNesting + + This needs to be marked accordingly. In the presentation, it might + make sense keep (3) separate, have (2) a sublist of (1). (1) can + be a closing tag, in which case (3) makes no sense at all, OR it + should be related with its opening tag (this may not necessarily + be possible before MakeWellFormed is run). + + So, the line and column counts as our identifier, so: + + $errors[$line][$col] = ... + + Then, we need to identify case 1, 2 or 3. They are identified as + such: + + 1. Need some sort of semaphore in RemoveForeignElements, etc. + 2. If CurrentAttr/CurrentCssProperty is non-null + 3. Default (FixNesting, MakeWellFormed) + + One consideration about (1) is that it usually is actually a + (3) modification, but we have no way of knowing about that because + of various optimizations. However, they can probably be treated + the same. The other difficulty is that (3) is never a line and + column; rather, it is a range (i.e. a duple) and telling the user + the very start of the range may confuse them. For example, + + Foo
bar
 + ^ ^ + + The node being operated on is , so the error would be assigned + to the first caret, with a "node reorganized" error. Then, the + ChildDef would have submitted its own suggestions and errors with + regard to what's going in the internals. So I suppose this is + ok. :-) + + Now, the structure of the earlier mentioned ... would be something + like this: + + object { + type = (token|attr|property), + value, // appropriate for type + errors => array(), + sub-errors = [recursive], + } + + This helps us keep things agnostic. It is also sufficiently complex + enough to warrant an object. + +So, more wanking about the object format is in order. The way HTML Purifier is +currently setup, the only possible hierarchy is: + + token -> attr -> css property + +These relations do not exist all of the time; a comment or end token would not +ever have any attributes, and non-style attributes would never have CSS properties +associated with them. + +I believe that it is worth supporting multiple paths. At some point, we might +have a hierarchy like: + + * -> syntax + -> token -> attr -> css property + -> url + -> css stylesheet + + + +

HTML align attribute to CSS

+ +

Inspect source for methodology.

+ +
+
+ HTML +
+
+ CSS +
+
+ +
+ +

table.align

+ +

left

+
+
+ a
O
a +
+
+ a
O
a +
+
+ +

center

+
+
+ a
O
a +
+
+ a
O
a +
+
+ +

right

+
+
+ a
O
a +
+
+ a
O
a +
+
+ +
+ + + +
+

img.align

+

left

+
+
+ aa +
+
+ aa +
+
+ +

right

+
+
+ aa +
+
+ aa +
+
+ +

bottom

+
+
+ aa +
+
+ aa +
+
+ +

middle

+
+
+ aa +
+
+ aa +
+
+ +

top

+
+
+ aa +
+
+ aa +
+
+ +
+ + + +
+ +

hr.align

+ +

left

+
+
+
+
+
+
+
+
+ +

center

+
+
+
+
+
+
+
+
+ +

right

+
+
+
+
+
+
+
+
+ +
+ + + diff --git a/vendor/ezyang/htmlpurifier/docs/specimens/img.png b/vendor/ezyang/htmlpurifier/docs/specimens/img.png new file mode 100644 index 0000000000..a755bcb5ed Binary files /dev/null and b/vendor/ezyang/htmlpurifier/docs/specimens/img.png differ diff --git a/vendor/ezyang/htmlpurifier/docs/specimens/jochem-blok-word.html b/vendor/ezyang/htmlpurifier/docs/specimens/jochem-blok-word.html new file mode 100644 index 0000000000..1cc08f8888 --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/specimens/jochem-blok-word.html @@ -0,0 +1,129 @@ + + + + + + + + + + + + +
+ +

+ +

 

+ +

Name

+ +

E-mail : mail@example.com

+ +

 

+ +

Company

+ +

Address 1

+ +

Address 2

+ +

 

+ +

Telefoon  : +xx xx xxx xxx xx

+ +

Fax  : +xx xx xxx xx xx

+ +

Internet : http://www.example.com

+ +

Kamer van koophandel +xxxxxxxxx

+ +

 

+ +

Op deze +e-mail is een disclaimer van toepassing, ga naar www.example.com/disclaimer
+A disclaimer is applicable to this email, please +refer to www.example.com/disclaimer

+ +

 

+ +
+ + + + diff --git a/vendor/ezyang/htmlpurifier/docs/specimens/windows-live-mail-desktop-beta.html b/vendor/ezyang/htmlpurifier/docs/specimens/windows-live-mail-desktop-beta.html new file mode 100644 index 0000000000..735b4bd95e --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/specimens/windows-live-mail-desktop-beta.html @@ -0,0 +1,74 @@ + + + + + + + +
Play +slideshow | Download the highest quality version of a picture by +clicking the + above it
+
+
    +
  1. Angry smile emoticonUn ka Tev iet, un ko tu dari? +
  2. Aha!
+ +
+ + + + + + +
+ +
+
This + is title for this + picture
 + +
+
 
+
Online +pictures are available for 30 days. Get Windows Live Mail desktop to create +your own photo e-mails.
diff --git a/vendor/ezyang/htmlpurifier/docs/style.css b/vendor/ezyang/htmlpurifier/docs/style.css new file mode 100644 index 0000000000..bd79c8a00d --- /dev/null +++ b/vendor/ezyang/htmlpurifier/docs/style.css @@ -0,0 +1,76 @@ +html {font-size:1em; font-family:serif; } +body {margin-left:4em; margin-right:4em; } + +dt {font-weight:bold; } +pre {margin-left:2em; } +pre, code, tt {font-family:monospace; font-size:1em; } + +h1 {text-align:center; font-family:Garamond, serif; + font-variant:small-caps;} +h2 {border-bottom:1px solid #CCC; font-family:sans-serif; font-weight:normal; + font-size:1.3em;} +h3 {font-family:sans-serif; font-size:1.1em; font-weight:bold; } +h4 {font-family:sans-serif; font-size:0.9em; font-weight:bold; } + +/* For witty quips */ +.subtitled {margin-bottom:0em;} +.subtitle , .subsubtitle {font-size:.8em; margin-bottom:1em; + font-style:italic; margin-top:-.2em;text-align:center;} +.subsubtitle {text-align:left;margin-left:2em;} + +/* Used for special "See also" links. */ +.reference {font-style:italic;margin-left:2em;} + +/* Marks off asides, discussions on why something is the way it is */ +.aside {margin-left:2em; font-family:sans-serif; font-size:0.9em; } +blockquote .label {font-weight:bold; font-size:1em; margin:0 0 .1em; + border-bottom:1px solid #CCC;} +.emphasis {font-weight:bold; text-align:center; font-size:1.3em;} + +/* A regular table */ +.table {border-collapse:collapse; border-bottom:2px solid #888; margin-left:2em; } +.table thead th {margin:0; background:#888; color:#FFF; } +.table thead th:first-child {-moz-border-radius-topleft:1em;} +.table tbody td {border-bottom:1px solid #CCC; padding-right:0.6em;padding-left:0.6em;} + +/* A quick table*/ +table.quick tbody th {text-align:right; padding-right:1em;} + +/* Category of the file */ +#filing {font-weight:bold; font-size:smaller; } + +/* Contains, without exception, Return to index. */ +#index {font-size:smaller; } + +#home {font-size:smaller;} + +/* Contains, without exception, $Id$, for SVN version info. */ +#version {text-align:right; font-style:italic; margin:2em 0;} + +#toc ol ol {list-style-type:lower-roman;} +#toc ol {list-style-type:decimal;} +#toc {list-style-type:upper-alpha;} + +q { + behavior: url(fixquotes.htc); /* IE fix */ + quotes: '\201C' '\201D' '\2018' '\2019'; +} +q:before { + content: open-quote; +} +q:after { + content: close-quote; +} + +/* Marks off implementation details interesting only to the person writing + the class described in the spec. */ +.technical {margin-left:2em; } +.technical:before {content:"Technical note: "; font-weight:bold; color:#061; } + +/* Marks off sections that are lacking. */ +.fixme {margin-left:2em; } +.fixme:before {content:"Fix me: "; font-weight:bold; color:#C00; } + +#applicability {margin: 1em 5%; font-style:italic;} + +/* vim: et sw=4 sts=4 */ -- cgit v1.3