Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion .github/workflows/nightly-locale-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,5 +64,16 @@ jobs:
!~/.m2/repository/org/apache/tika
key: maven-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
restore-keys: maven-${{ runner.os }}-
# Test failures do not stop the reactor, so every module runs and one night shows every
# failure for the locale; the next step fails the job from the reports.
- name: Build with Maven under ${{ matrix.locale }}
run: mvn install -Pfast -pl :tika-annotation-processor -am -B -q && mvn clean test install -Pci -T1C -Dtika.test.locale=${{ matrix.locale }} -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
run: mvn install -Pfast -pl :tika-annotation-processor -am -B -q && mvn clean test install -Pci -T1C -Dmaven.test.failure.ignore=true -Dtika.test.locale=${{ matrix.locale }} -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
- name: Fail on any test failure
run: |
reports=$(find . -path '*/target/surefire-reports/TEST-*.xml' -o -path '*/target/failsafe-reports/TEST-*.xml')
failed=$(awk '/<testcase /{tc=$0} /<(failure|error)[ >]/{match(tc, /name="[^"]*"/); n=substr(tc, RSTART+6, RLENGTH-7); match(tc, /classname="[^"]*"/); c=substr(tc, RSTART+11, RLENGTH-12); print " " c "." n}' $reports | sort -u)
if [ -n "$failed" ]; then
echo "Test failures under ${{ matrix.locale }}:"
echo "$failed"
exit 1
fi
42 changes: 42 additions & 0 deletions tika-core/src/test/java/org/apache/tika/test/TestLocalesTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.test;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.Locale;

import org.junit.jupiter.api.Test;

public class TestLocalesTest {

@Test
public void testCalendarVariantsAreNonGregorian() {
// plain th-TH too: Calendar.getInstance gives th_TH a Buddhist calendar
assertTrue(TestLocales.nonGregorian(Locale.forLanguageTag("th-TH")));
assertTrue(TestLocales.nonGregorian(Locale.forLanguageTag("th-TH-u-nu-thai-x-lvariant-TH")));
assertTrue(TestLocales.nonGregorian(Locale.forLanguageTag("ja-JP-u-ca-japanese-x-lvariant-JP")));
}

@Test
public void testOrdinaryLocalesAreGregorian() {
for (String tag : new String[]{"th", "ja-JP", "ar-EG", "tr-TR", "de-DE", "en-US"}) {
assertFalse(TestLocales.nonGregorian(Locale.forLanguageTag(tag)), tag);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@
import java.util.Locale;
import java.util.regex.Pattern;

import org.apache.poi.util.LocaleUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Isolated;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -38,7 +42,29 @@
* broadly similar output.
* This is mostly focused on the XHTML output
*/
//pins the JVM default locale, which is global: no other class may run alongside
@Isolated
public class TabularFormatsTest extends TikaTest {

//POI's DataFormatter and parso's DataWriterUtil emit the default locale's digits whatever
//locale they are given; pin the default until that is fixed upstream (TIKA-4920)
private static Locale savedLocale;
private static Locale savedPoiLocale;

@BeforeAll
public static void pinLocale() {
savedLocale = Locale.getDefault();
savedPoiLocale = LocaleUtil.getUserLocale();
Locale.setDefault(Locale.US);
LocaleUtil.setUserLocale(Locale.US);
}

@AfterAll
public static void restoreLocale() {
LocaleUtil.setUserLocale(savedPoiLocale);
Locale.setDefault(savedLocale);
}

protected static final String[] columnNames =
new String[]{"recnum", "square", "desc", "pctdone", "pctincr", "date", "datetime",
"time"};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.test.TestLocales;

/**
* Test case for parsing various different font files.
Expand Down Expand Up @@ -88,8 +89,11 @@ public void testTTFParsing() throws Exception {
assertEquals("application/x-font-ttf", metadata.get(HttpHeaders.CONTENT_TYPE));
assertEquals("Open Sans Bold", metadata.get(TikaCoreProperties.TITLE));

assertEquals("2010-12-30T11:04:00Z", metadata.get(TikaCoreProperties.CREATED));
assertEquals("2011-05-05T12:37:53Z", metadata.get(TikaCoreProperties.MODIFIED));
// fontbox TTFDataStream.readInternationalDate builds its Calendar in the default locale (TIKA-4920)
if (!TestLocales.nonGregorianDefault()) {
assertEquals("2010-12-30T11:04:00Z", metadata.get(TikaCoreProperties.CREATED));
assertEquals("2011-05-05T12:37:53Z", metadata.get(TikaCoreProperties.MODIFIED));
}

assertEquals("Open Sans Bold", metadata.get(MET_FONT_NAME));
assertEquals("Open Sans", metadata.get(MET_FONT_FAMILY_NAME));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.text.DateFormat;
import java.text.DateFormatSymbols;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
Expand Down Expand Up @@ -94,7 +93,7 @@ public void oneOff() throws Exception {
try {
Date d = MailDateParser.parseDateLenient(s);
DateFormat df =
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateFormatSymbols(Locale.US));
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
df.setTimeZone(TimeZone.getTimeZone("UTC"));
String dateString = df.format(d);
System.out.println("dev parser lenient: " + dateString);
Expand Down Expand Up @@ -130,7 +129,7 @@ public void testDates() throws Exception {

@Test
public void testTrickyDates() throws Exception {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd", new DateFormatSymbols(Locale.US));
DateFormat df = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
//make sure there are no mis-parses of e.g. 90 = year 90 A.D, not 1990
Date date1980 = df.parse("1980-01-01");
Date date2010 = df.parse("2010-01-01");
Expand Down Expand Up @@ -180,7 +179,7 @@ private void testDate(String dateString, String expected, boolean useUTC) throws
Date parsedDate = MailDateParser.parseDateLenient(dateString);
assertNotNull(parsedDate, "couldn't parse " + dateString);
DateFormat df =
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateFormatSymbols(Locale.US));
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
if (useUTC) {
df.setTimeZone(TimeZone.getTimeZone("UTC"));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.test.TestLocales;
import org.apache.tika.utils.XMLReaderUtils;

/**
Expand Down Expand Up @@ -241,11 +242,14 @@ public void testOutlookHTMLVersion() throws Exception {
metadata.get(MAPI.SUBMISSION_ID));
assertEquals("<EBB9951D34EA4B41B70AB946CF3FB6EC1A297D98@ftm02.FT.FTG.COM>",
metadata.get(MAPI.INTERNET_MESSAGE_ID));
assertTrue(metadata.get(MAPI.SUBMISSION_ACCEPTED_AT_TIME).startsWith("2011-03-29"));
assertTrue(metadata.get(MAPI.CLIENT_SUBMIT_TIME).startsWith("2011-03-29"));
assertTrue(metadata.get("mapi:message-delivery-time").startsWith("2011-03-29"));
assertTrue(metadata.get("mapi:last-modification-time").startsWith("2011-03-29"));
assertTrue(metadata.get("mapi:creation-time").startsWith("2011-03-29"));
// POI LocaleUtil.getLocaleCalendar dates MAPI times in the default calendar (TIKA-4920)
if (!TestLocales.nonGregorianDefault()) {
assertTrue(metadata.get(MAPI.SUBMISSION_ACCEPTED_AT_TIME).startsWith("2011-03-29"));
assertTrue(metadata.get(MAPI.CLIENT_SUBMIT_TIME).startsWith("2011-03-29"));
assertTrue(metadata.get("mapi:message-delivery-time").startsWith("2011-03-29"));
assertTrue(metadata.get("mapi:last-modification-time").startsWith("2011-03-29"));
assertTrue(metadata.get("mapi:creation-time").startsWith("2011-03-29"));
}
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.test.TestLocales;

/**
* Test case for parsing rar files.
Expand Down Expand Up @@ -74,9 +75,12 @@ public void testEmbedded() throws Exception {
for (String crt : tracker.createdAts) {
assertNull(crt);
}
for (String mod : tracker.modifiedAts) {
assertNotNull(mod);
assertTrue(mod.startsWith("20"), "Modified at " + mod);
// junrar FileHeader.getDateDos uses Calendar.getInstance() (TIKA-4920)
if (!TestLocales.nonGregorianDefault()) {
for (String mod : tracker.modifiedAts) {
assertNotNull(mod);
assertTrue(mod.startsWith("20"), "Modified at " + mod);
}
}

// Should have filenames in the content string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledIf;
import org.junit.jupiter.api.parallel.Isolated;
import org.junit.jupiter.api.parallel.ResourceLock;
import org.junit.jupiter.api.parallel.Resources;
Expand All @@ -43,6 +44,8 @@
*/
@Isolated
@ResourceLock(Resources.TIME_ZONE)
// commons-compress ZipUtil.dosToJavaDate uses Calendar.getInstance() (TIKA-4920)
@DisabledIf("org.apache.tika.test.TestLocales#nonGregorianDefault")
public class ZipDosTimeDstTest extends TikaTest {

private static TimeZone saved;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.test.TestLocales;

/** Runs in a +14h zone. */
@Isolated
Expand Down Expand Up @@ -86,7 +87,10 @@ public void testEntryTimesAreFileSystemTimes(@TempDir Path tempDir) throws Excep
List<Metadata> list = getRecursiveMetadata(zip, new ParseContext(), false);
Metadata dosMd = entry(list, "dos.txt");
Metadata extMd = entry(list, "ext.txt");
assertEquals("2009-08-11T09:09:44", dosMd.get(FileSystem.MODIFIED));
// commons-compress ZipUtil.dosToJavaDate uses Calendar.getInstance() (TIKA-4920)
if (!TestLocales.nonGregorianDefault()) {
assertEquals("2009-08-11T09:09:44", dosMd.get(FileSystem.MODIFIED));
}
assertEquals("2012-02-20T16:44:22Z", extMd.get(FileSystem.MODIFIED));
for (Metadata m : List.of(dosMd, extMd)) {
assertNull(m.get(TikaCoreProperties.CREATED));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.test;

import java.util.Calendar;
import java.util.Locale;
import java.util.TimeZone;

/** Conditions for {@code @DisabledIf} / {@code @EnabledIf} on locale-sensitive tests. */
public final class TestLocales {

private TestLocales() {
}

/**
* True when the default locale's calendar is not Gregorian (th-TH and its legacy variant,
* ja-JP-u-ca-japanese-x-lvariant-JP). Libraries that build dates from
* {@code Calendar.getInstance()} then set era-relative years. Guard only the date assertions,
* {@code if (!TestLocales.nonGregorianDefault()) { ... }}, with a comment naming the library;
* use {@code @DisabledIf("org.apache.tika.test.TestLocales#nonGregorianDefault")} only when
* every assertion is such a date.
*/
public static boolean nonGregorianDefault() {
return nonGregorian(Locale.getDefault());
}

static boolean nonGregorian(Locale locale) {
// not instanceof GregorianCalendar: BuddhistCalendar extends it
return !"gregory".equals(Calendar.getInstance(TimeZone.getTimeZone("UTC"), locale).getCalendarType());
}
}
Loading