Fix multibyte character problem

Fixed problem that causes a UnicodeEncodeError when use multi-byte characters in the job.yml. Change-Id: Ie715c827a794e73fae11cdca079ea80cfb8c280d
12 years ago · 604d39305b
parent 777a29aa8a
commit 604d39305b
42 changed files with 61 additions and 40 deletions
--- a/jenkins_jobs/builder.py
+++ b/jenkins_jobs/builder.py
@ -381,7 +381,7 @@ class XmlJob(object):

    def output(self):
        out = minidom.parseString(XML.tostring(self.xml))
-        out = out.toprettyxml(indent='  ')
+        out = out.toprettyxml(indent='  ', encoding='utf-8')
        return self.pretty_text_re.sub('>\g<1></', out)


--- a/tests/base.py
+++ b/tests/base.py
@ -17,6 +17,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.

+import codecs
 import os
 import re
 import doctest
@ -61,7 +62,7 @@ class BaseTestCase(object):
    def __read_content(self):
        # Read XML content, assuming it is unicode encoded
        xml_filepath = os.path.join(self.fixtures_path, self.xml_filename)
-        xml_content = u"%s" % open(xml_filepath, 'r').read()
+        xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()

        yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
        with file(yaml_filepath, 'r') as yaml_file:
@ -83,7 +84,8 @@ class BaseTestCase(object):
        pub.gen_xml(parser, xml_project, yaml_content)

        # Prettify generated XML
-        pretty_xml = XmlJob(xml_project, 'fixturejob').output()
+        pretty_xml = unicode(XmlJob(xml_project, 'fixturejob').output(),
+                             'utf-8')

        self.assertThat(
            pretty_xml,
--- a/tests/builders/fixtures/maven-target-doc.xml
+++ b/tests/builders/fixtures/maven-target-doc.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <builders>
    <hudson.tasks.Maven>
--- a/tests/builders/fixtures/maven-target001.xml
+++ b/tests/builders/fixtures/maven-target001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <builders>
    <hudson.tasks.Maven>
--- a/tests/builders/fixtures/trigger-builds001.xml
+++ b/tests/builders/fixtures/trigger-builds001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <builders>
    <hudson.plugins.parameterizedtrigger.TriggerBuilder>
--- a/tests/general/fixtures/assigned-node001.xml
+++ b/tests/general/fixtures/assigned-node001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <actions/>
  <keepDependencies>false</keepDependencies>
--- a/tests/general/fixtures/assigned-node002.xml
+++ b/tests/general/fixtures/assigned-node002.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <actions/>
  <keepDependencies>false</keepDependencies>
--- a/tests/notifications/fixtures/http-endpoint001.xml
+++ b/tests/notifications/fixtures/http-endpoint001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <properties>
    <com.tikal.hudson.plugins.notification.HudsonNotificationProperty>
--- a/tests/parameters/fixtures/dynamic-choice-param001.xml
+++ b/tests/parameters/fixtures/dynamic-choice-param001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <properties>
    <hudson.model.ParametersDefinitionProperty>
--- a/tests/properties/fixtures/batch-task.xml
+++ b/tests/properties/fixtures/batch-task.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <properties>
    <hudson.plugins.batch__task.BatchTaskProperty>
--- a/tests/properties/fixtures/throttle001.xml
+++ b/tests/properties/fixtures/throttle001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <properties>
    <hudson.plugins.throttleconcurrents.ThrottleJobProperty>
--- a/tests/publishers/fixtures/README
+++ b/tests/publishers/fixtures/README
@ -10,5 +10,5 @@ Each yaml file MUST have a corresponding xml file.

 Once the YAML file has been parsed, it is prettify using python minidom
 which also means that:
- your XML file must start with: <?xml version="1.0" ?>
+- your XML file must start with: <?xml version="1.0" encoding="utf-8"?>
 - self closing elements do not contains space eg: <element/>
--- a/tests/publishers/fixtures/archive001.xml
+++ b/tests/publishers/fixtures/archive001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
    <publishers>
        <hudson.tasks.ArtifactArchiver>
--- a/tests/publishers/fixtures/blame001.xml
+++ b/tests/publishers/fixtures/blame001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.blame__upstream__commiters.BlameUpstreamCommitersPublisher/>
--- a/tests/publishers/fixtures/build001.xml
+++ b/tests/publishers/fixtures/build001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.build__publisher.BuildPublisher>
--- a/tests/publishers/fixtures/cloverphp001.xml
+++ b/tests/publishers/fixtures/cloverphp001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <org.jenkinsci.plugins.cloverphp.CloverPHPPublisher>
--- a/tests/publishers/fixtures/cloverphp002.xml
+++ b/tests/publishers/fixtures/cloverphp002.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <org.jenkinsci.plugins.cloverphp.CloverPHPPublisher>
--- a/tests/publishers/fixtures/cloverphp003.xml
+++ b/tests/publishers/fixtures/cloverphp003.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <org.jenkinsci.plugins.cloverphp.CloverPHPPublisher>
--- a/tests/publishers/fixtures/description-setter001.xml
+++ b/tests/publishers/fixtures/description-setter001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
--- a/tests/publishers/fixtures/description-setter001.yaml
+++ b/tests/publishers/fixtures/description-setter001.yaml
--- a/tests/publishers/fixtures/description-setter002.xml
+++ b/tests/publishers/fixtures/description-setter002.xml
@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <publishers>
+    <hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
+      <regexp>.*(&lt;a href=.*a&gt;)</regexp>
+      <regexpForFailed>.*(&lt;a href=.*a&gt;)</regexpForFailed>
+      <description>こんにちは</description>
+      <descriptionForFailed>さようなら</descriptionForFailed>
+      <setForMatrix>true</setForMatrix>
+    </hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
+  </publishers>
+</project>
--- a/tests/publishers/fixtures/description-setter002.yaml
+++ b/tests/publishers/fixtures/description-setter002.yaml
@ -0,0 +1,7 @@
+publishers:
+  - description-setter:
+      regexp: ".*(<a href=.*a>)"
+      regexp-for-failed: ".*(<a href=.*a>)"
+      description: "こんにちは"
+      description-for-failed: "さようなら"
+      set-for-matrix: true
--- a/tests/publishers/fixtures/emotional-jenkins.xml
+++ b/tests/publishers/fixtures/emotional-jenkins.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <org.jenkinsci.plugins.emotional__jenkins.EmotionalJenkinsPublisher/>
--- a/tests/publishers/fixtures/git001.xml
+++ b/tests/publishers/fixtures/git001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.git.GitPublisher>
--- a/tests/publishers/fixtures/github-notifier.xml
+++ b/tests/publishers/fixtures/github-notifier.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <com.cloudbees.jenkins.GitHubCommitNotifier/>
--- a/tests/publishers/fixtures/mavendeploy001.xml
+++ b/tests/publishers/fixtures/mavendeploy001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.maven.RedeployPublisher>
--- a/tests/publishers/fixtures/pipeline001.xml
+++ b/tests/publishers/fixtures/pipeline001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <au.com.centrumsystems.hudson.plugin.buildpipeline.trigger.BuildPipelineTrigger>
--- a/tests/publishers/fixtures/pipeline002.xml
+++ b/tests/publishers/fixtures/pipeline002.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <au.com.centrumsystems.hudson.plugin.buildpipeline.trigger.BuildPipelineTrigger>
--- a/tests/publishers/fixtures/plot001.xml
+++ b/tests/publishers/fixtures/plot001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.plot.PlotPublisher>
--- a/tests/publishers/fixtures/plot002.xml
+++ b/tests/publishers/fixtures/plot002.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.plot.PlotPublisher>
--- a/tests/publishers/fixtures/plot003.xml
+++ b/tests/publishers/fixtures/plot003.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <hudson.plugins.plot.PlotPublisher>
--- a/tests/publishers/fixtures/scp001.xml
+++ b/tests/publishers/fixtures/scp001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <be.certipost.hudson.plugin.SCPRepositoryPublisher>
--- a/tests/publishers/fixtures/stash001.xml
+++ b/tests/publishers/fixtures/stash001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <org.jenkinsci.plugins.stashNotifier.StashNotifier>
--- a/tests/publishers/fixtures/xunit001.xml
+++ b/tests/publishers/fixtures/xunit001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <publishers>
    <xunit>
--- a/tests/scm/fixtures/git-shallow-clone01.xml
+++ b/tests/scm/fixtures/git-shallow-clone01.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <scm class="hudson.plugins.git.GitSCM">
    <configVersion>2</configVersion>
--- a/tests/scm/fixtures/git-shallow-clone02.xml
+++ b/tests/scm/fixtures/git-shallow-clone02.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <scm class="hudson.plugins.git.GitSCM">
    <configVersion>2</configVersion>
--- a/tests/scm/fixtures/git-shallow-clone03.xml
+++ b/tests/scm/fixtures/git-shallow-clone03.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <scm class="hudson.plugins.git.GitSCM">
    <configVersion>2</configVersion>
--- a/tests/scm/fixtures/gitlab.xml
+++ b/tests/scm/fixtures/gitlab.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <scm class="hudson.plugins.git.GitSCM">
    <configVersion>2</configVersion>
--- a/tests/scm/fixtures/repo001.xml
+++ b/tests/scm/fixtures/repo001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <scm class="hudson.plugins.repo.RepoScm">
    <manifestRepositoryUrl>https://example.com/project/</manifestRepositoryUrl>
--- a/tests/triggers/fixtures/gerrit001.xml
+++ b/tests/triggers/fixtures/gerrit001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <triggers class="vector">
    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
--- a/tests/triggers/fixtures/github-pull-request.xml
+++ b/tests/triggers/fixtures/github-pull-request.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <triggers class="vector">
    <org.jenkinsci.plugins.ghprb.GhprbTrigger>
--- a/tests/wrappers/fixtures/env-file001.xml
+++ b/tests/wrappers/fixtures/env-file001.xml
@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
  <buildWrappers>
    <hudson.plugins.envfile.EnvFileBuildWrapper>