diff --git a/sdk-java/src/main/java/ly/count/sdk/java/Config.java b/sdk-java/src/main/java/ly/count/sdk/java/Config.java index 56275a98..852c5214 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/Config.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/Config.java @@ -253,6 +253,26 @@ public class Config { protected boolean locationEnabled = true; protected boolean autoSendUserProperties = true; + /** + * Seed value for the SDK behavior settings. Used on first init only, + * when no settings have yet been persisted; subsequent inits use the + * value pulled from storage. + */ + protected String sdkBehaviorSettings = null; + + /** + * If {@code true}, the SDK skips the periodic {@code /o/sdk?method=sc} + * fetch. The seed JSON from {@link #setSdkBehaviorSettings(String)} (or + * a previously-persisted snapshot) is still applied. + */ + protected boolean sdkBehaviorSettingsRequestsDisabled = false; + + /** + * Whether the backoff mechanism may hold the request queue back after the server answered + * slowly. The SDK behavior settings can override it. + */ + protected boolean backoffMechanismEnabled = true; + // TODO: storage limits & configuration // protected int maxRequestsStored = 0; // protected int storageDirectory = ""; @@ -1508,4 +1528,42 @@ public Config disableAutoSendUserProperties() { this.autoSendUserProperties = false; return this; } + + /** + * Provide an initial SDK behavior settings payload to apply at init time, + * before the first {@code /o/sdk?method=sc} fetch returns. The value is + * the same JSON shape the Countly server emits: {@code {"v":1,"t":, + * "c":{...}}}. The seed is only used when nothing has been persisted yet. + * + * @param sdkBehaviorSettings serialized config JSON, or {@code null} for none + * @return {@code this} instance for method chaining + */ + public Config setSdkBehaviorSettings(String sdkBehaviorSettings) { + this.sdkBehaviorSettings = sdkBehaviorSettings; + return this; + } + + /** + * Disable the periodic SDK behavior settings fetch from the server. Any + * seed JSON or previously-persisted snapshot is still applied at init. + * + * @return {@code this} instance for method chaining + */ + public Config disableSdkBehaviorSettingsUpdates() { + this.sdkBehaviorSettingsRequestsDisabled = true; + return this; + } + + /** + * Disable the backoff mechanism. While it is on, a request the server takes long to answer + * makes the SDK wait before it sends the next queued request, which gives a busy server room + * to recover. It never waits while many requests are queued or after an old request. The SDK + * behavior settings can still turn it on or off. + * + * @return {@code this} instance for method chaining + */ + public Config disableBackoffMechanism() { + this.backoffMechanismEnabled = false; + return this; + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java new file mode 100644 index 00000000..65631d52 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java @@ -0,0 +1,425 @@ +package ly.count.sdk.java.internal; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * The SDK behavior settings in effect, the server side configuration served by + * {@code /o/sdk?method=sc}. Every value starts from the developer configuration and is overridden, + * in increasing order of precedence, by the settings provided at init (used only while nothing is + * stored), by the settings stored from an earlier run and by the latest server response. + *

+ * {@link ModuleConfiguration} implements it for the running SDK. Reach it through + * {@link InternalConfig#getConfigurationProvider()}, which never returns {@code null}. A server + * response can change any value on the thread that delivered it, so read a value where it is used + * instead of keeping a copy of it. + */ +public interface ConfigurationProvider { + + /** + * Key {@code networking}. + * + * @return whether requests may be sent at all, the queued ones and the immediate ones + */ + boolean getNetworkingEnabled(); + + /** + * Key {@code tracking}. + * + * @return whether anything new may be recorded + */ + boolean getTrackingEnabled(); + + /** + * Key {@code st}. + * + * @return whether sessions may be tracked + */ + boolean getSessionTrackingEnabled(); + + /** + * Key {@code vt}. + * + * @return whether views may be tracked + */ + boolean getViewTrackingEnabled(); + + /** + * Key {@code cet}. + * + * @return whether custom events, the ones whose key does not start with {@code [CLY]_}, may be recorded + */ + boolean getCustomEventTrackingEnabled(); + + /** + * Key {@code ecz}. + * + * @return whether the SDK should be in the content zone + */ + boolean getContentZoneEnabled(); + + /** + * Key {@code crt}. + * + * @return whether crashes may be recorded + */ + boolean getCrashReportingEnabled(); + + /** + * Key {@code ast}. This SDK has no automatic session tracking, so the value is only exposed. + * + * @return whether sessions should be tracked automatically + */ + boolean getAutomaticSessionTrackingEnabled(); + + /** + * Key {@code avt}. This SDK has no automatic view tracking, so the value is only exposed. + * + * @return whether views should be tracked automatically + */ + boolean getAutomaticViewTrackingEnabled(); + + /** + * Key {@code acr}. + * + * @return whether unhandled exceptions should be reported automatically + */ + boolean getAutomaticCrashReportingEnabled(); + + /** + * Key {@code lt}. + * + * @return whether location may be tracked + */ + boolean getLocationTrackingEnabled(); + + /** + * Key {@code rcz}. + * + * @return whether the content zone may be refreshed + */ + boolean getRefreshContentZoneEnabled(); + + /** + * Key {@code bom}. + * + * @return whether the backoff mechanism is on + */ + boolean getBOMEnabled(); + + /** + * Key {@code log}. + * + * @return whether the SDK should print its logs + */ + boolean getLoggingEnabled(); + + /** + * Key {@code cr}. The SDK applies it when it initializes, so a value that arrives later takes + * effect on the next init and can differ from {@link InternalConfig#requiresConsent()} until then. + * + * @return whether consent is required before anything is recorded + */ + boolean getConsentRequired(); + + /** + * Key {@code scui}. + * + * @return hours between two fetches of the settings + */ + int getServerConfigUpdateInterval(); + + /** + * Key {@code rqs}. + * + * @return the maximum number of queued requests + */ + int getRequestQueueMaxSize(); + + /** + * Whether {@link #getRequestQueueMaxSize()} comes from the settings, provided, stored or + * received, rather than from the developer configuration. + * + * @return {@code true} when the settings set the request queue size + */ + boolean isRequestQueueMaxSizeFromBehaviorSettings(); + + /** + * Key {@code eqs}. + * + * @return how many events are held before they are sent + */ + int getEventQueueSizeThreshold(); + + /** + * Key {@code sui}. + * + * @return seconds between two ticks of the SDK timer, which sends session updates and queued events + */ + int getSessionUpdateInterval(); + + /** + * Key {@code lkl}. + * + * @return the maximum length of a key, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxKeyLength(); + + /** + * Key {@code lvs}. + * + * @return the maximum length of a value, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxValueSize(); + + /** + * Key {@code lsv}. + * + * @return the maximum number of segmentation entries, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxSegmentationValues(); + + /** + * Key {@code lbc}. + * + * @return the maximum number of breadcrumbs kept for a crash + */ + int getMaxBreadcrumbCount(); + + /** + * Key {@code ltlpt}. + * + * @return the maximum number of stack trace lines per thread, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxStackTraceLinesPerThread(); + + /** + * Key {@code ltl}. + * + * @return the maximum length of a stack trace line, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxStackTraceLineLength(); + + /** + * Key {@code upcl}. + * + * @return the maximum number of custom user properties cached before they are sent, + * {@link Integer#MAX_VALUE} for no limit + */ + int getUserPropertyCacheLimit(); + + /** + * Key {@code bom_at}. + * + * @return seconds a request may take before the backoff mechanism counts it as slow + */ + int getBOMAcceptedTimeoutSeconds(); + + /** + * Key {@code bom_rqp}. + * + * @return the share of the request queue, in (0, 1), above which the backoff mechanism stays off + */ + double getBOMRQPercentage(); + + /** + * Key {@code bom_ra}. + * + * @return the age, in hours, above which a request is too old for the backoff mechanism to delay it + */ + int getBOMRequestAge(); + + /** + * Key {@code bom_d}. + * + * @return seconds the backoff mechanism holds the request queue back + */ + int getBOMDuration(); + + /** + * Key {@code dort}. + * + * @return the age, in hours, above which a queued request is dropped, {@code 0} to keep every request + */ + int getRequestDropAgeHours(); + + /** + * Key {@code czi}. + * + * @return seconds between two content fetches while in the content zone + */ + int getContentZoneTimerInterval(); + + /** + * Keys {@code eb} and {@code ew}. + * + * @return the filter for custom event keys, never {@code null} + */ + @Nonnull FilterList> getEventFilterList(); + + /** + * Keys {@code upb} and {@code upw}. + * + * @return the filter for custom user property keys, never {@code null} + */ + @Nonnull FilterList> getUserPropertyFilterList(); + + /** + * Keys {@code sb} and {@code sw}. + * + * @return the filter for custom event segmentation keys, never {@code null} + */ + @Nonnull FilterList> getSegmentationFilterList(); + + /** + * Keys {@code esb} and {@code esw}. + * + * @return the segmentation key filter of each custom event key, never {@code null} + */ + @Nonnull FilterList>> getEventSegmentationFilterList(); + + /** + * Key {@code jte}. + * + * @return the custom event keys that trigger a journey, never {@code null} and never modifiable + */ + @Nonnull Set getJourneyTriggerEvents(); + + /** + * Key {@code jtv}. + * + * @return the view names that trigger a journey, never {@code null} and never modifiable + */ + @Nonnull Set getJourneyTriggerViews(); + + /** + * The decision of the {@code lg} directive, which sits next to {@code c} in a live response. + * + * @return whether this device gathers its SDK logs for the server + */ + @Nonnull LogGatheringState getLogGatheringState(); + + /** + * The gather id every uploaded log batch has to carry. + * + * @return the id, {@code null} unless {@link #getLogGatheringState()} is {@link LogGatheringState#GATHERING} + */ + @Nullable String getLogGatheringId(); + + /** + * The log levels to gather, as level characters: e error, w warning, i info, d debug, v verbose. + * + * @return a non empty subset of {@code ewidv} + */ + @Nonnull String getLogGatheringLevels(); + + /** + * How many log lines to hold before a batch is uploaded. + * + * @return the batch size, within [10, 500] + */ + int getLogGatheringBatchSize(); + + /** + * Whether this device gathers its SDK logs for the server. + */ + enum LogGatheringState { + /** + * No live response has decided yet: only stored or provided settings were seen. + */ + UNDECIDED, + /** + * A live response enabled gathering and carried a usable gather id. + */ + GATHERING, + /** + * A live response, a failed fetch or disabled settings requests decided against gathering. + */ + NOT_GATHERING + } + + /** + * An immutable listing filter: names that are the only ones allowed, a whitelist, or the ones + * rejected, a blacklist. An empty blacklist allows everything. + * + * @param the names, as a set or as a set per event key + */ + final class FilterList { + static final FilterList> NO_NAMES = new FilterList<>(Collections.emptySet(), false); + static final FilterList>> NO_NAMES_PER_EVENT = new FilterList<>(Collections.>emptyMap(), false); + + private final T filterList; + private final boolean isWhitelist; + + /** + * Wraps names that are already immutable. + * + * @param filterList the names + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + */ + private FilterList(@Nonnull T filterList, boolean isWhitelist) { + this.filterList = filterList; + this.isWhitelist = isWhitelist; + } + + /** + * Builds a filter over an immutable copy of the given names. + * + * @param names the names, kept in the given order + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + * @return the filter + */ + static FilterList> ofNames(@Nonnull Set names, boolean isWhitelist) { + return new FilterList<>(Collections.unmodifiableSet(new LinkedHashSet<>(names)), isWhitelist); + } + + /** + * Builds a filter over an immutable copy of the given names of each event key. + * + * @param namesPerEvent the names of each event key, kept in the given order + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + * @return the filter + */ + static FilterList>> ofNamesPerEvent(@Nonnull Map> namesPerEvent, boolean isWhitelist) { + Map> copy = new LinkedHashMap<>(); + for (Map.Entry> entry : namesPerEvent.entrySet()) { + copy.put(entry.getKey(), Collections.unmodifiableSet(new LinkedHashSet<>(entry.getValue()))); + } + return new FilterList<>(Collections.unmodifiableMap(copy), isWhitelist); + } + + /** + * The names of the filter. + * + * @return the names, never modifiable + */ + @Nonnull + public T getFilterList() { + return filterList; + } + + /** + * Whether the names are the only ones allowed. + * + * @return {@code true} for a whitelist, {@code false} for a blacklist + */ + public boolean isWhitelist() { + return isWhitelist; + } + + /** + * Describes the filter for logs. + * + * @return the kind of the filter followed by its names + */ + @Override + public String toString() { + return (isWhitelist ? "whitelist " : "blacklist ") + filterList; + } + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java index f2dd2205..f663c614 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java @@ -8,6 +8,7 @@ public class CountlyTimer { private final Log L; private ScheduledExecutorService timerService; + private volatile long timerDelaySeconds = -1; protected static int TIMER_DELAY_MS = 0; // for testing purposes protected CountlyTimer(Log logger) { @@ -53,6 +54,7 @@ protected void startTimer(long timerDelay, Runnable runnable) { */ protected void startTimer(long timerDelay, long initialDelayMs, Runnable runnable) { L.i("[CountlyTimer] startTimer, Starting global timer timerDelay: [" + timerDelay + "] initialDelayMs: [" + initialDelayMs + "]"); + timerDelaySeconds = timerDelay; long delay = timerDelay * 1000; if (delay < 1000) { @@ -68,4 +70,13 @@ protected void startTimer(long timerDelay, long initialDelayMs, Runnable runnabl timerService.scheduleWithFixedDelay(runnable, startTime, delay, TimeUnit.MILLISECONDS); } + + /** + * The interval this timer was last started with, as it was requested. + * + * @return the interval in seconds, {@code -1} while the timer was never started + */ + long getTimerDelaySeconds() { + return timerDelaySeconds; + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java index 72e6003e..802aa0c9 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java @@ -6,9 +6,11 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; import ly.count.sdk.java.Crash; import org.json.JSONException; import org.json.JSONObject; @@ -23,6 +25,10 @@ public class CrashImpl implements Crash, Storable { private final JSONObject data; private Throwable throwable; private Map traces; + private Thread tracesMainThread; + //whether "_error" holds the dump of addTraces rather than the stack trace of a throwable + private boolean errorIsTraceDump = false; + private String[] logs; protected CrashImpl(Log logger) { this(TimeUtils.uniqueTimestampMs(), logger); @@ -38,6 +44,7 @@ protected CrashImpl(Long id, Log logger) { @Override public CrashImpl addThrowable(Throwable throwable) { this.throwable = throwable; + this.errorIsTraceDump = false; StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -57,31 +64,58 @@ public CrashImpl addTraces(Thread main, Map traces) return this; } else { this.traces = traces; + String dump = printAllTraces(main, traces, Integer.MAX_VALUE, "[CrashImpl] addTraces"); + this.tracesMainThread = main; + this.errorIsTraceDump = true; + return add("_type", "anr").add("_error", dump); + } + } - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); + /** + * Prints the stack traces of every thread, the main thread first. + * + * @param main the main thread, {@code null} for none + * @param threadTraces the stack trace of each thread + * @param maxLinesPerThread how many of the top lines of each thread to print at most + * @param tag the caller, for the log of a thread that has more lines + * @return the printed stack traces + */ + private String printAllTraces(Thread main, @Nonnull Map threadTraces, int maxLinesPerThread, @Nonnull String tag) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); - if (main != null && traces.containsKey(main)) { - pw.println("Thread [main]:"); - printTraces(pw, null, traces.get(main)); - pw.append("\n\n"); - } + if (main != null && threadTraces.containsKey(main)) { + pw.println("Thread [main]:"); + printTraces(pw, main, true, threadTraces.get(main), maxLinesPerThread, tag); + pw.append("\n\n"); + } - for (Thread thread : traces.keySet()) { - if (thread != main) { - printTraces(pw, thread, traces.get(thread)); - pw.append("\n\n"); - } + for (Thread thread : threadTraces.keySet()) { + if (thread != main) { + printTraces(pw, thread, false, threadTraces.get(thread), maxLinesPerThread, tag); + pw.append("\n\n"); } - return add("_type", "anr").add("_error", sw.toString()); } + return sw.toString(); } - private void printTraces(PrintWriter pw, Thread thread, StackTraceElement[] traces) { - if (thread != null) { + /** + * Prints the stack trace of one thread, under a header naming the thread unless it is the main one. + * + * @param pw where to print + * @param thread the thread + * @param mainThread whether it is the main thread, whose header is printed by the caller + * @param traces the stack trace of the thread + * @param maxLines how many of the top lines to print at most + * @param tag the caller, for the log when lines are left out + */ + private void printTraces(PrintWriter pw, Thread thread, boolean mainThread, StackTraceElement[] traces, int maxLines, @Nonnull String tag) { + if (!mainThread && thread != null) { pw.append("Thread [").append(thread.getName()).append("]:\n"); } - for (StackTraceElement el : traces) { + int lines = UtilsInternalLimits.stackTraceLinesToKeep(traces.length, maxLines, thread == null ? null : thread.getName(), L, tag); + for (int i = 0; i < lines; i++) { + StackTraceElement el = traces[i]; pw.append("\tat ").append(el == null ? "<>" : el.toString()).append("\n"); } } @@ -111,11 +145,90 @@ protected CrashImpl addSegments(Map segments) { @Override public CrashImpl setLogs(String[] logs) { if (logs != null && logs.length > 0) { + this.logs = logs.clone(); return add("_logs", Utils.join(Arrays.asList(logs), "\n")); } return this; } + /** + * Applies the SDK internal limits of the SDK behavior settings to what this crash sends: the lines + * per thread of a dump made by {@link #addTraces(Thread, Map)}, the length of every stack trace + * line, the keys, string values and number of the custom segments and the length of every + * breadcrumb. The stack trace of a throwable keeps all its lines. Nothing changes while no limit + * is exceeded. + * + * @param limits the settings in effect + * @param tag the caller, for the log + */ + void applyInternalLimits(@Nonnull ConfigurationProvider limits, @Nonnull String tag) { + int maxLinesPerThread = limits.getMaxStackTraceLinesPerThread(); + if (errorIsTraceDump && traceDumpExceeds(maxLinesPerThread)) { + add("_error", printAllTraces(tracesMainThread, traces, maxLinesPerThread, tag)); + } + + String error = data.optString("_error", null); + if (error != null) { + String truncatedError = UtilsInternalLimits.truncateStackTraceLines(error, limits.getMaxStackTraceLineLength(), L, tag); + if (truncatedError.length() != error.length()) { + add("_error", truncatedError); + } + } + + JSONObject custom = data.optJSONObject("_custom"); + if (custom != null) { + Map segments = new LinkedHashMap<>(); + for (String key : custom.keySet()) { + segments.put(key, custom.opt(key)); + } + Map limitedSegments = UtilsInternalLimits.applySegmentationLimits(segments, limits, L, tag); + if (!segments.equals(limitedSegments)) { + add("_custom", new JSONObject(limitedSegments)); + } + } + + applyValueSizeLimitToLogs(limits.getMaxValueSize(), tag); + } + + /** + * Whether a thread of the dump made by {@link #addTraces(Thread, Map)} has more stack trace lines + * than a limit. + * + * @param maxLinesPerThread the limit + * @return {@code true} when a thread has more lines + */ + private boolean traceDumpExceeds(int maxLinesPerThread) { + for (StackTraceElement[] threadTraces : traces.values()) { + if (threadTraces != null && threadTraces.length > maxLinesPerThread) { + return true; + } + } + return false; + } + + /** + * Cuts every breadcrumb given to {@link #setLogs(String[])} to the value size limit. + * + * @param maxValueSize the limit + * @param tag the caller, for the log + */ + private void applyValueSizeLimitToLogs(int maxValueSize, @Nonnull String tag) { + if (logs == null) { + return; + } + + String[] truncatedLogs = new String[logs.length]; + boolean truncated = false; + for (int i = 0; i < logs.length; i++) { + truncatedLogs[i] = UtilsInternalLimits.truncateValue(logs[i], maxValueSize, L, tag); + truncated |= logs[i] != null && truncatedLogs[i].length() != logs[i].length(); + } + + if (truncated) { + setLogs(truncatedLogs); + } + } + @Override public Throwable getThrowable() { return throwable; diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java index 085775c3..c0633d1b 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java @@ -1,20 +1,52 @@ package ly.count.sdk.java.internal; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public class DefaultNetworking implements Networking { + /** + * Name of the thread that ends a backoff of the request queue. + */ + static final String BACKOFF_THREAD_NAME = "network-backoff"; + private static final String TIMESTAMP_PARAM = "timestamp"; + private static final long MS_IN_HOUR = 60L * 60L * 1000L; + private Log L = null; private Transport transport; private Tasks tasks; - private boolean shutdown; + private volatile boolean shutdown; + private volatile boolean backedOff; + private boolean backendMode; + private ScheduledExecutorService backoffScheduler; IStorageForRequestQueue storageForRequestQueue; + /** + * Prepares the transport, the thread requests are sent on and the scheduler that ends a + * backoff, which starts its daemon thread only when a first backoff is scheduled. + * + * @param config configuration of the SDK being initialized + * @param storageForRequestQueue the request queue to drain + */ @Override public void init(InternalConfig config, IStorageForRequestQueue storageForRequestQueue) { L = config.getLogger(); shutdown = false; + backedOff = false; + backendMode = config.isBackendModeEnabled(); transport = new Transport(); transport.init(config); tasks = new Tasks("network", L); + backoffScheduler = Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, BACKOFF_THREAD_NAME); + thread.setDaemon(true); + return thread; + }); this.storageForRequestQueue = storageForRequestQueue; } @@ -23,20 +55,44 @@ public boolean isSending() { return tasks.isRunning(); } + /** + * Starts draining the request queue unless the SDK stopped, a request is being handled, there is + * no device ID yet or the backoff mechanism holds the queue back. + * + * @param config configuration of the running SDK + * @return whether a request is being handled + */ @Override public boolean check(InternalConfig config) { - L.d("[Networking] [check] state: shutdown [" + shutdown + "], tasks running [" + tasks.isRunning() + "], net running [" + tasks.isRunning() + "], device id [" + config.getDeviceId() + "]"); - if (!shutdown && !tasks.isRunning() && config.getDeviceId() != null) { + L.d("[Networking] [check] state: shutdown [" + shutdown + "], backed off [" + backedOff + "], tasks running [" + tasks.isRunning() + "], net running [" + tasks.isRunning() + "], device id [" + config.getDeviceId() + "]"); + if (!shutdown && !backedOff && !tasks.isRunning() && config.getDeviceId() != null) { tasks.run(submit(config)); } return tasks.isRunning(); } + /** + * Sends the oldest queued request, unless networking is off or the backoff mechanism holds the + * queue back. Queued requests older than the request drop age are dropped first, unsent. Once + * the server accepts the request the next one follows, unless the server answered so slowly + * that the backoff mechanism holds the queue back. + * + * @param config configuration of the running SDK + * @return the task, resolving to whether a request was handed over for sending or removed + */ protected Tasks.Task submit(final InternalConfig config) { return new Tasks.Task(Tasks.ID_STRICT) { @Override public Boolean call() throws Exception { - final Request request = storageForRequestQueue.getNextRequest(); + if (!config.getNetworkingEnabled()) { + L.d("[Networking] submit, networking disabled by SDK behavior settings; skipping queue drain"); + return false; + } + if (backedOff) { + L.d("[Networking] submit, the backoff mechanism holds the request queue back; skipping queue drain"); + return false; + } + final Request request = nextRequestWithinDropAge(config); if (request == null) { return false; } else { @@ -54,11 +110,14 @@ public Boolean call() throws Exception { request.params.remove("rr"); } request.params.add("rr", storageForRequestQueue.remaningRequests()); - tasks.run(transport.send(request), result -> { - L.d("[Networking] Request " + request.storageId() + " sent?: " + result); + final AtomicLong responseTimeMs = new AtomicLong(-1L); + tasks.run(transport.send(request, responseTimeMs::set), result -> { + L.d("[Networking] Request " + request.storageId() + " sent?: " + result + ", response time [" + responseTimeMs.get() + "] ms"); if (result) { storageForRequestQueue.removeRequest(request); - check(config); + if (!backOffIfTheServerIsSlow(config, request, responseTimeMs.get())) { + check(config); + } } }); return true; @@ -68,9 +127,208 @@ public Boolean call() throws Exception { }; } + /** + * Reads the oldest queued request, first dropping, unsent, every queued request older than the + * request drop age of the SDK behavior settings. + * + * @param config configuration of the running SDK + * @return the request to send next, {@code null} when the queue is empty or a request could not be dropped + */ + @Nullable + private Request nextRequestWithinDropAge(@Nonnull InternalConfig config) { + Request request = storageForRequestQueue.getNextRequest(); + while (request != null && isOlderThanDropAge(config, request)) { + if (!dropWithoutSending(config, request)) { + return null; + } + request = storageForRequestQueue.getNextRequest(); + } + return request; + } + + /** + * Whether a queued request is older than the request drop age of the SDK behavior settings, a + * drop age of {@code 0} keeping every request. Never in backend mode, where the settings are + * inert and a request carries the time the application recorded it at. + * + * @param config configuration of the running SDK + * @param request the queued request + * @return {@code true} when the request has to be dropped without being sent + */ + private boolean isOlderThanDropAge(@Nonnull InternalConfig config, @Nonnull Request request) { + if (backendMode) { + return false; + } + + int dropAgeHours = config.getConfigurationProvider().getRequestDropAgeHours(); + return dropAgeHours > 0 && requestAgeMs(request) > dropAgeHours * MS_IN_HOUR; + } + + /** + * Removes a queued request without sending it, then tells the module that owns it, with a + * {@code null} response and {@link Transport#NO_RESPONSE_CODE}, so nothing keeps waiting for + * its response. + * + * @param config configuration of the running SDK + * @param request the request to drop + * @return whether the request was removed + */ + private boolean dropWithoutSending(@Nonnull InternalConfig config, @Nonnull Request request) { + L.w("[Networking] dropWithoutSending, request [" + request.storageId() + "] is older than the request drop age of [" + config.getConfigurationProvider().getRequestDropAgeHours() + "] hours set by the SDK behavior settings, dropping it without sending"); + if (!Boolean.TRUE.equals(storageForRequestQueue.removeRequest(request))) { + L.e("[Networking] dropWithoutSending, failed to remove request [" + request.storageId() + "], the queue drain stops here"); + return false; + } + + Class requestOwner = request.owner(); + SDKCore core = SDKCore.instance; + if (requestOwner != null && core != null) { + try { + core.onRequestCompleted(request, null, Transport.NO_RESPONSE_CODE, requestOwner); + } catch (Exception e) { + L.e("[Networking] dropWithoutSending, failed to tell the owner of request [" + request.storageId() + "] that it was dropped, [" + e + "]"); + } + } + return true; + } + + /** + * How long ago a request was recorded, read from its {@code timestamp} parameter, or from its + * storage ID, which is taken from the clock when the request is created, when it has none. + * + * @param request the request + * @return the age in milliseconds, negative for a request dated in the future + */ + private long requestAgeMs(@Nonnull Request request) { + Long recordedAtMs = request.storageId(); + String timestamp = request.params == null ? null : request.params.get(TIMESTAMP_PARAM); + if (timestamp != null) { + try { + recordedAtMs = Long.parseLong(timestamp); + } catch (NumberFormatException e) { + L.w("[Networking] requestAgeMs, request [" + request.storageId() + "] has a timestamp that is not a number [" + timestamp + "], dating it by its storage ID"); + } + } + + long nowMs = System.currentTimeMillis(); + return recordedAtMs == null ? 0L : nowMs - recordedAtMs; + } + + /** + * Holds the request queue back for the backoff duration of the SDK behavior settings when the + * backoff mechanism is on and the server answered slowly: the response took at least the + * accepted timeout, the request is not older than the backoff request age, and the requests + * still queued are at most the given share of the request queue size. Never in backend mode, + * where the settings are inert. + * + * @param config configuration of the running SDK + * @param request the request the server just accepted, already removed from the queue + * @param responseTimeMs how long the server took to answer it, in milliseconds + * @return whether the queue is now held back + */ + private boolean backOffIfTheServerIsSlow(@Nonnull InternalConfig config, @Nonnull Request request, long responseTimeMs) { + if (backendMode) { + return false; + } + + ConfigurationProvider provider = config.getConfigurationProvider(); + if (!provider.getBOMEnabled() || responseTimeMs < provider.getBOMAcceptedTimeoutSeconds() * 1000L) { + return false; + } + + if (requestAgeMs(request) > provider.getBOMRequestAge() * MS_IN_HOUR) { + L.v("[Networking] backOffIfTheServerIsSlow, the server took [" + responseTimeMs + "] ms to answer request [" + request.storageId() + "], but the request is older than [" + provider.getBOMRequestAge() + "] hours, not backing off"); + return false; + } + + int queuedRequests = queuedRequestCount(); + if (queuedRequests < 0) { + return false; + } + + double queueShare = provider.getRequestQueueMaxSize() * provider.getBOMRQPercentage(); + if (queuedRequests > queueShare) { + L.v("[Networking] backOffIfTheServerIsSlow, the server took [" + responseTimeMs + "] ms to answer request [" + request.storageId() + "], but [" + queuedRequests + "] requests remain queued, more than [" + queueShare + "], not backing off"); + return false; + } + + int durationSeconds = provider.getBOMDuration(); + L.i("[Networking] backOffIfTheServerIsSlow, the server took [" + responseTimeMs + "] ms to answer request [" + request.storageId() + "] and [" + queuedRequests + "] requests remain queued, holding the request queue back for [" + durationSeconds + "] seconds"); + return startBackoff(config, durationSeconds); + } + + /** + * How many requests the queue holds. + * + * @return the number of queued requests, {@code -1} when they could not be counted + */ + private int queuedRequestCount() { + try { + Integer remaining = storageForRequestQueue.remaningRequests(); + return remaining == null ? -1 : remaining + 1; + } catch (RuntimeException e) { + L.w("[Networking] queuedRequestCount, failed to count the queued requests, not backing off, [" + e + "]"); + return -1; + } + } + + /** + * Holds the request queue back for the given time, after which it drains again. + * + * @param config configuration of the running SDK + * @param durationSeconds how long to hold the queue back + * @return whether the queue is held back, {@code false} when the SDK is stopping + */ + private boolean startBackoff(@Nonnull final InternalConfig config, int durationSeconds) { + backedOff = true; + try { + backoffScheduler.schedule(() -> endBackoff(config), durationSeconds, TimeUnit.SECONDS); + return true; + } catch (RejectedExecutionException e) { + backedOff = false; + L.d("[Networking] startBackoff, the SDK is stopping, the request queue is not held back"); + return false; + } + } + + /** + * Ends a backoff and lets the request queue drain again, unless the SDK stopped meanwhile. + * + * @param config configuration of the running SDK + */ + private void endBackoff(@Nonnull InternalConfig config) { + backedOff = false; + if (shutdown) { + return; + } + + L.d("[Networking] endBackoff, the backoff is over, the request queue drains again"); + try { + check(config); + } catch (RuntimeException e) { + L.w("[Networking] endBackoff, failed to restart the request queue drain, [" + e + "]"); + } + } + + /** + * Stops sending. A backoff that is still running never ends in a drain, and a request that is + * being sent may finish. + * + * @param config configuration of the SDK being stopped + */ @Override public void stop(InternalConfig config) { shutdown = true; + if (backoffScheduler != null) { + backoffScheduler.shutdownNow(); + try { + if (!backoffScheduler.awaitTermination(1, TimeUnit.SECONDS)) { + L.w("[Networking] stop, the backoff scheduler did not stop in time"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } tasks.shutdown(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java index d446315d..8d633c1d 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nonnull; import ly.count.sdk.java.Config; /** @@ -32,6 +33,11 @@ public class InternalConfig extends Config { protected IdGenerator viewIdGenerator; protected IdGenerator eventIdGenerator; protected ViewIdProvider viewIdProvider; + /** + * The SDK behavior settings of the running SDK, registered by {@link ModuleConfiguration} when it + * initializes. Read it through {@link #getConfigurationProvider()}. + */ + volatile ConfigurationProvider configProvider; /** * Shouldn't be used! @@ -144,13 +150,47 @@ public void setDefaultNetworking(boolean defaultNetworking) { } /** - * This feature is not yet implemented - * and always return true + * The SDK behavior settings in effect. While no {@link ModuleConfiguration} is registered on + * this configuration, as for a module built on its own in a test, the settings come from this + * configuration alone. + * + * @return the settings, never {@code null} + */ + @Nonnull + ConfigurationProvider getConfigurationProvider() { + ConfigurationProvider provider = configProvider; + if (provider != null) { + return provider; + } + return ModuleConfiguration.developerDefaults(this); + } + + /** + * Whether the SDK is currently allowed to perform network requests, the {@code networking} + * switch of the SDK behavior settings. * - * @return true + * @return {@code true} if networking is allowed */ public boolean getNetworkingEnabled() { - return true; + return getConfigurationProvider().getNetworkingEnabled(); + } + + public String getSdkBehaviorSettings() { + return sdkBehaviorSettings; + } + + public boolean isSdkBehaviorSettingsRequestsDisabled() { + return sdkBehaviorSettingsRequestsDisabled; + } + + /** + * Whether the developer left the backoff mechanism on, the value the {@code bom} setting of the + * SDK behavior settings starts from. + * + * @return {@code false} after {@link Config#disableBackoffMechanism()} + */ + boolean isBackoffMechanismEnabled() { + return backoffMechanismEnabled; } /** diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java index 9fb214ea..de2f26f9 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.nio.file.Files; import javax.annotation.Nonnull; +import org.json.JSONArray; import org.json.JSONObject; public class JsonFileStorage { @@ -32,19 +33,27 @@ public JsonFileStorage(@Nonnull final File file, @Nonnull Log logger) { * @param key to set * @param value to add */ - public void add(@Nonnull final String key, @Nonnull Object value) { + public synchronized void add(@Nonnull final String key, @Nonnull Object value) { logger.i("[JsonFileStorage] add, Adding key: [" + key + "], value: [" + value + "]"); json.put(key, value); } /** - * Saves changes to the disk/db/memory + * Saves changes to the disk/db/memory. When the data cannot be serialized, the file keeps what + * it had. */ - public void save() { + public synchronized void save() { logger.i("[JsonFileStorage] save, Saving json file: [" + file.getAbsolutePath() + "]"); + // Serialized before the file is opened: opening truncates it, and toString returns null instead of throwing + String content = json.toString(); + if (content == null) { + logger.e("[JsonFileStorage] save, Failed to serialize the data, the json file is left as it is: [" + file.getAbsolutePath() + "]"); + return; + } + try (BufferedWriter writer = Files.newBufferedWriter(file.toPath())) { - writer.write(json.toString()); + writer.write(content); } catch (IOException e) { logger.e("[JsonFileStorage] save, Failed to save json file, reason: [" + e.getMessage() + "]"); } @@ -57,7 +66,7 @@ public void save() { * * @param key to remove */ - public void delete(@Nonnull final String key) { + public synchronized void delete(@Nonnull final String key) { if (!json.has(key)) { logger.v("[JsonFileStorage] delete, Nothing to delete"); } @@ -71,7 +80,7 @@ public void delete(@Nonnull final String key) { * @param key to set * @param value to add */ - public void addAndSave(@Nonnull final String key, @Nonnull Object value) { + public synchronized void addAndSave(@Nonnull final String key, @Nonnull Object value) { add(key, value); save(); } @@ -82,7 +91,7 @@ public void addAndSave(@Nonnull final String key, @Nonnull Object value) { * * @param key to remove */ - public void deleteAndSave(@Nonnull final String key) { + public synchronized void deleteAndSave(@Nonnull final String key) { delete(key); save(); } @@ -94,19 +103,53 @@ public void deleteAndSave(@Nonnull final String key) { * @param key to get * @return value */ - public Object get(@Nonnull final String key) { + public synchronized Object get(@Nonnull final String key) { return json.opt(key); } /** - * Returns JSONObject value for the key + * Returns a copy of the JSONObject value for the key. Changing the copy changes nothing stored: + * hand it back through {@link #add(String, Object)} to store it, so another thread that saves + * never serializes an object while it is being changed. * * @param key to get * @param defaultValue to return if key not found - * @return value, if key not found returns defaultValue + * @return a deep copy of the value, if key not found returns defaultValue */ - public JSONObject getJsonObj(@Nonnull final String key, final JSONObject defaultValue) { - return json.optJSONObject(key, defaultValue); + public synchronized JSONObject getJsonObj(@Nonnull final String key, final JSONObject defaultValue) { + JSONObject value = json.optJSONObject(key, null); + if (value == null) { + return defaultValue; + } + return (JSONObject) deepCopy(value); + } + + /** + * Copies a JSON value, every nested object and array included, keeping every other value as it is. + * + * @param value the value to copy + * @return the copy, or the value itself when it holds nothing that can change + */ + private static Object deepCopy(final Object value) { + if (value instanceof JSONObject) { + JSONObject source = (JSONObject) value; + JSONObject copy = new JSONObject(); + for (String key : source.keySet()) { + copy.put(key, deepCopy(source.opt(key))); + } + return copy; + } + + if (value instanceof JSONArray) { + JSONArray source = (JSONArray) value; + JSONArray copy = new JSONArray(); + for (int i = 0; i < source.length(); i++) { + copy.put(deepCopy(source.opt(i))); + } + return copy; + } + + return value; } /** @@ -115,7 +158,7 @@ public JSONObject getJsonObj(@Nonnull final String key, final JSONObject default * @param key to get * @return value, if key not found returns null */ - public String getString(@Nonnull final String key) { + public synchronized String getString(@Nonnull final String key) { return getString(key, null); } @@ -126,7 +169,7 @@ public String getString(@Nonnull final String key) { * @param defaultValue to return if key not found * @return value, if key not found returns defaultValue */ - public String getString(@Nonnull final String key, final String defaultValue) { + public synchronized String getString(@Nonnull final String key, final String defaultValue) { return json.optString(key, defaultValue); } @@ -137,21 +180,21 @@ public String getString(@Nonnull final String key, final String defaultValue) { * @param defaultValue to return if key not found * @return value, if key not found returns defaultValue */ - public int getInt(@Nonnull final String key, final int defaultValue) { + public synchronized int getInt(@Nonnull final String key, final int defaultValue) { return json.optInt(key, defaultValue); } /** * Clears all data */ - public void clear() { + public synchronized void clear() { json.clear(); } /** * Clears all data and saves changes to the disk/db/memory */ - public void clearAndSave() { + public synchronized void clearAndSave() { clear(); save(); } @@ -161,7 +204,7 @@ public void clearAndSave() { * * @return number of key-value pairs */ - public int size() { + public synchronized int size() { return json.length(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java index 2fd4a8f7..7a5abf4c 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java @@ -10,15 +10,44 @@ public class Log { private final LogCallback logListener; private final Config.LoggingLevel loggingLevel; + //the lowest level printed: the developer level unless the SDK behavior settings override it + private volatile Config.LoggingLevel printLevel; public Log(Config.LoggingLevel loggingLevel, LogCallback logListener) { if (loggingLevel == null) { throw new NullPointerException("Logging level can't null."); } this.loggingLevel = loggingLevel; + this.printLevel = loggingLevel; this.logListener = logListener; } + /** + * Applies the {@code log} switch of the SDK behavior settings to what is printed. Switched off, + * nothing is printed. Switched on, the developer level is printed, or every level when the + * developer level is {@link Config.LoggingLevel#OFF}. The log listener receives every line either way. + * + * @param loggingEnabled the resolved {@code log} switch + */ + void setLoggingEnabled(boolean loggingEnabled) { + if (!loggingEnabled) { + printLevel = Config.LoggingLevel.OFF; + } else if (loggingLevel == Config.LoggingLevel.OFF) { + printLevel = Config.LoggingLevel.VERBOSE; + } else { + printLevel = loggingLevel; + } + } + + /** + * The lowest level printed, which the SDK behavior settings can change. + * + * @return the level, {@link Config.LoggingLevel#OFF} while nothing is printed + */ + Config.LoggingLevel getPrintLevel() { + return printLevel; + } + /** * {@link Config.LoggingLevel} level logging * @@ -70,7 +99,7 @@ public void v(String logMessage) { } private void print(String msg, Config.LoggingLevel level) { - if (level != null && loggingLevel.prints(level)) { + if (level != null && printLevel.prints(level)) { System.out.println(msg); } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java index 2dfc65d4..42e2256c 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java @@ -79,7 +79,7 @@ private void recordEventInternal(String deviceID, String key, int count, Double eventQueues.get(deviceID).put(jsonObject); ++eventQSize; - if (eventQSize >= internalConfig.getEventsBufferSize()) { + if (eventQSize >= internalConfig.getConfigurationProvider().getEventQueueSizeThreshold()) { addEventsToRequestQ(); } } @@ -319,7 +319,7 @@ private synchronized void addEventsToRequestQ() { private void addRequestToRequestQ(Request request) { synchronized (SDKCore.instance.lockBRQStorage) { L.d("[BackendMode] addRequestToRequestQ"); - if (internalConfig.getRequestQueueMaxSize() == SDKCore.instance.requestQueueMemory.size()) { + if (internalConfig.getConfigurationProvider().getRequestQueueMaxSize() == SDKCore.instance.requestQueueMemory.size()) { L.d("[BackendMode] addRequestToRequestQ: In Memory request queue is full, dropping oldest request: " + request.params.toString()); SDKCore.instance.requestQueueMemory.remove(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java index 0b676ace..0ee557e2 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java @@ -95,9 +95,16 @@ public Boolean onRequest(Request request) { } /** - * Called when the request is executed. + * Called when the request is executed, only on the owning module ({@link Request#own(Class)}), + * once for every attempt to send it, on the networking thread. A request dropped unsent for + * being older than the request drop age gets a call too, with a {@code null} response and + * {@link Transport#NO_RESPONSE_CODE}. * Gives the module the ability to respond to the response received * Request identification should be done through the Request ID + * + * @param request the request that was sent + * @param response the response body, {@code null} when none could be read + * @param responseCode the response code, {@link Transport#NO_RESPONSE_CODE} when no response arrived */ public void onRequestCompleted(Request request, String response, int responseCode) { @@ -124,4 +131,17 @@ protected void initFinished(InternalConfig config) { protected void deviceIdChanged(String oldDeviceId, boolean withMerge) { } + + /** + * Called when a server response changed the SDK behavior settings while the SDK runs, after + * the SDK applied what it owns: the logger, the timer interval and the request queue. Read the + * new values through {@link InternalConfig#getConfigurationProvider()}. Runs on the thread that + * delivered the response, which is usually not the thread that initialized the SDK, and can run + * after {@link #init(InternalConfig)} but before {@link #initFinished(InternalConfig)} when the + * response of the fetch made at init arrives that early. + * + * @param config configuration of the running SDK + */ + protected void onSdkConfigurationChanged(InternalConfig config) { + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java new file mode 100644 index 00000000..e5e9f2f4 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java @@ -0,0 +1,1340 @@ +package ly.count.sdk.java.internal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import ly.count.sdk.java.Config; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Resolves, stores and serves the SDK behavior settings, the server side configuration served by + * {@code /o/sdk?method=sc}. + *

+ * Every setting starts from the developer configuration and is overridden by the settings provided + * at init (only while nothing is stored), by the stored settings and by every later server response. + * {@link SDKCore} runs {@link #init(InternalConfig)}, which does the resolve, before it starts the + * global timer and builds the other modules; {@link #initFinished(InternalConfig)} fetches the + * settings and starts the refresh timer. In backend mode the module stays inert and serves the + * developer configuration. + *

+ * Resolved values are published through volatile fields and immutable snapshots, so a reader on any + * thread sees consistent values while a response is applied on the thread that delivered it. + */ +public class ModuleConfiguration extends ModuleBase implements ConfigurationProvider { + + static final String keyRTimestamp = "t"; + static final String keyRVersion = "v"; + static final String keyRConfig = "c"; + static final String keyRLogGathering = "lg"; + static final String keyRConnectionTest = "ct"; + + static final String keyRTracking = "tracking"; + static final String keyRNetworking = "networking"; + static final String keyRReqQueueSize = "rqs"; + static final String keyREventQueueSize = "eqs"; + static final String keyRLogging = "log"; + static final String keyRSessionUpdateInterval = "sui"; + static final String keyRSessionTracking = "st"; + static final String keyRViewTracking = "vt"; + static final String keyRLocationTracking = "lt"; + static final String keyRRefreshContentZone = "rcz"; + static final String keyRLimitKeyLength = "lkl"; + static final String keyRLimitValueSize = "lvs"; + static final String keyRLimitSegValues = "lsv"; + static final String keyRLimitBreadcrumb = "lbc"; + static final String keyRLimitTraceLine = "ltlpt"; + static final String keyRLimitTraceLength = "ltl"; + static final String keyRCustomEventTracking = "cet"; + static final String keyREnterContentZone = "ecz"; + static final String keyRContentZoneInterval = "czi"; + static final String keyRConsentRequired = "cr"; + static final String keyRDropOldRequestTime = "dort"; + static final String keyRCrashReporting = "crt"; + static final String keyRAutomaticSessionTracking = "ast"; + static final String keyRAutomaticViewTracking = "avt"; + static final String keyRAutomaticCrashReporting = "acr"; + static final String keyRServerConfigUpdateInterval = "scui"; + static final String keyRBackoffMechanism = "bom"; + static final String keyRBOMAcceptedTimeout = "bom_at"; + static final String keyRBOMRQPercentage = "bom_rqp"; + static final String keyRBOMRequestAge = "bom_ra"; + static final String keyRBOMDuration = "bom_d"; + static final String keyRUserPropertyCacheLimit = "upcl"; + static final String keyREventBlacklist = "eb"; + static final String keyRUserPropertyBlacklist = "upb"; + static final String keyRSegmentationBlacklist = "sb"; + static final String keyREventSegmentationBlacklist = "esb"; + static final String keyREventWhitelist = "ew"; + static final String keyRUserPropertyWhitelist = "upw"; + static final String keyRSegmentationWhitelist = "sw"; + static final String keyREventSegmentationWhitelist = "esw"; + static final String keyRJourneyTriggerEvents = "jte"; + static final String keyRJourneyTriggerViews = "jtv"; + + static final String keyLGEnabled = "e"; + static final String keyLGId = "i"; + static final String keyLGLevels = "l"; + static final String keyLGBatchSize = "b"; + + static final String logGatheringAllLevels = "ewidv"; + static final int logGatheringDefaultBatchSize = 100; + static final int logGatheringMinBatchSize = 10; + //the log buffer ceiling, so also the largest batch the server may ask for: a bigger batch would never fill + static final int logGatheringMaxBufferedLines = 500; + + static final int DEFAULT_SERVER_CONFIG_UPDATE_INTERVAL_HOURS = 4; + static final int DEFAULT_BOM_ACCEPTED_TIMEOUT_SECONDS = 10; + static final double DEFAULT_BOM_RQ_PERCENTAGE = 0.5; + static final int DEFAULT_BOM_REQUEST_AGE_HOURS = 24; + static final int DEFAULT_BOM_DURATION_SECONDS = 60; + static final int MIN_CONTENT_ZONE_INTERVAL = 16; + static final int NO_LIMIT = Integer.MAX_VALUE; + + //the settings merged from every accepted configuration, the stored form; guarded by this + JSONObject latestRetrievedConfigurationFull = null; + JSONObject latestRetrievedConfiguration = null; + + //guarded by this + CountlyTimer serverConfigUpdateTimer = null; + + private boolean serverConfigRequestsDisabled = false; + private volatile boolean running = false; + private volatile ConnectionTestListener connectionTestListener = null; + + //resolved values; the constant seeds are here, the ones read from the developer configuration are in seedFromDeveloperConfig + volatile boolean currentVTracking = true; + volatile boolean currentVNetworking = true; + volatile boolean currentVSessionTracking = true; + volatile boolean currentVViewTracking = true; + volatile boolean currentVCustomEventTracking = true; + volatile boolean currentVContentZone = false; + volatile boolean currentVCrashReporting = true; + volatile boolean currentVAutomaticSessionTracking = false; + volatile boolean currentVAutomaticViewTracking = false; + volatile boolean currentVAutomaticCrashReporting = true; + volatile boolean currentVLocationTracking = true; + volatile boolean currentVRefreshContentZone = true; + volatile boolean currentVBackoffMechanism = true; + volatile boolean currentVLoggingEnabled = false; + volatile boolean currentVRequiresConsent = false; + + volatile int currentVServerConfigUpdateInterval = DEFAULT_SERVER_CONFIG_UPDATE_INTERVAL_HOURS; + volatile int currentVRequestQueueMaxSize = 1000; + volatile boolean currentVRequestQueueMaxSizeFromBehaviorSettings = false; + volatile int currentVEventQueueSizeThreshold = 10; + volatile int currentVSessionUpdateInterval = 60; + volatile int currentVMaxKeyLength = NO_LIMIT; + volatile int currentVMaxValueSize = NO_LIMIT; + volatile int currentVMaxSegmentationValues = NO_LIMIT; + volatile int currentVMaxBreadcrumbCount = 100; + volatile int currentVMaxStackTraceLinesPerThread = NO_LIMIT; + volatile int currentVMaxStackTraceLineLength = NO_LIMIT; + volatile int currentVUserPropertyCacheLimit = NO_LIMIT; + volatile int currentVBOMAcceptedTimeoutSeconds = DEFAULT_BOM_ACCEPTED_TIMEOUT_SECONDS; + volatile double currentVBOMRQPercentage = DEFAULT_BOM_RQ_PERCENTAGE; + volatile int currentVBOMRequestAge = DEFAULT_BOM_REQUEST_AGE_HOURS; + volatile int currentVBOMDuration = DEFAULT_BOM_DURATION_SECONDS; + volatile int currentVDropAgeHours = 0; + volatile int currentVZoneTimerInterval = ConfigContent.DEFAULT_ZONE_TIMER_INTERVAL; + + volatile FilterList> currentVEventFilterList = FilterList.NO_NAMES; + volatile FilterList> currentVUserPropertyFilterList = FilterList.NO_NAMES; + volatile FilterList> currentVSegmentationFilterList = FilterList.NO_NAMES; + volatile FilterList>> currentVEventSegmentationFilterList = FilterList.NO_NAMES_PER_EVENT; + volatile Set currentVJourneyTriggerEvents = Collections.emptySet(); + volatile Set currentVJourneyTriggerViews = Collections.emptySet(); + volatile LogGatheringDirective currentVLogGathering = LogGatheringDirective.UNDECIDED; + + /** + * Runs the connection test that a live settings response asks for with its {@code ct} flag. + */ + interface ConnectionTestListener { + /** + * Called on the thread that delivered the response, after the settings in it were applied. + * + * @param fetchLatencyMs how long the settings request took, from sending it to its response, in milliseconds + */ + void onConnectionTestRequested(long fetchLatencyMs); + } + + /** + * One decision of the log gathering directive, published as a whole so a reader never sees the + * state of one directive next to the gather id of another. + */ + static final class LogGatheringDirective { + static final LogGatheringDirective UNDECIDED = new LogGatheringDirective(LogGatheringState.UNDECIDED, null, logGatheringAllLevels, logGatheringDefaultBatchSize); + static final LogGatheringDirective NOT_GATHERING = new LogGatheringDirective(LogGatheringState.NOT_GATHERING, null, logGatheringAllLevels, logGatheringDefaultBatchSize); + + final LogGatheringState state; + final String gatherId; + final String levels; + final int batchSize; + + /** + * Holds one decision. + * + * @param state what was decided + * @param gatherId the gather id, {@code null} unless gathering + * @param levels the level characters to gather + * @param batchSize how many lines make a batch + */ + LogGatheringDirective(@Nonnull LogGatheringState state, @Nullable String gatherId, @Nonnull String levels, int batchSize) { + this.state = state; + this.gatherId = gatherId; + this.levels = levels; + this.batchSize = batchSize; + } + } + + /** + * Builds a provider that serves the developer configuration alone, for code that reads the + * settings while no configuration module is registered on that configuration. + * + * @param config the developer configuration to serve + * @return a provider that never changes + */ + @Nonnull + static ConfigurationProvider developerDefaults(@Nonnull InternalConfig config) { + ModuleConfiguration defaults = new ModuleConfiguration(); + defaults.seedFromDeveloperConfig(config); + return defaults; + } + + /** + * Seeds every setting from the developer configuration and resolves the provided and the stored + * settings over it, then registers this module as the provider of the configuration. In backend + * mode nothing is loaded and the developer configuration stays in effect. + * + * @param config configuration of the SDK being initialized + */ + @Override + public void init(InternalConfig config) { + super.init(config); + boolean backendMode = config.isBackendModeEnabled(); + serverConfigRequestsDisabled = backendMode || config.isSdkBehaviorSettingsRequestsDisabled(); + L.d("[ModuleConfiguration] init, backend mode:[" + backendMode + "], server config requests disabled:[" + serverConfigRequestsDisabled + "]"); + + seedFromDeveloperConfig(config); + + if (backendMode) { + L.d("[ModuleConfiguration] init, backend mode is enabled, SDK behavior settings are not applied"); + } else { + loadConfigFromStorage(config.getSdkBehaviorSettings()); + } + + updateConfigVariables(config, null); + running = true; + config.configProvider = this; + } + + /** + * Fetches the settings from the server and starts the refresh timer, unless settings requests + * are disabled or backend mode is on. + * + * @param config configuration of the running SDK + */ + @Override + public void initFinished(@Nonnull InternalConfig config) { + L.d("[ModuleConfiguration] initFinished"); + if (serverConfigRequestsDisabled) { + return; + } + + fetchConfigFromServer(config); + startServerConfigUpdateTimer(); + } + + /** + * Stops the refresh timer; a response that arrives afterwards is ignored. + * + * @param config configuration of the SDK being stopped + * @param clear whether the SDK clears its data, which removes the stored settings with it + */ + @Override + public void stop(InternalConfig config, boolean clear) { + super.stop(config, clear); + running = false; + stopServerConfigUpdateTimer(); + } + + /** + * Sets the resolved values that come from the developer configuration. + * + * @param config the developer configuration + */ + private void seedFromDeveloperConfig(@Nonnull InternalConfig config) { + currentVLoggingEnabled = config.getLoggingLevel() != null && config.getLoggingLevel() != Config.LoggingLevel.OFF; + currentVRequiresConsent = config.requiresConsent(); + currentVAutomaticCrashReporting = config.isUnhandledCrashReportingEnabled(); + currentVBackoffMechanism = config.isBackoffMechanismEnabled(); + currentVRequestQueueMaxSize = config.getRequestQueueMaxSize(); + currentVEventQueueSizeThreshold = config.getEventsBufferSize(); + currentVSessionUpdateInterval = config.getSendUpdateEachSeconds(); + currentVMaxBreadcrumbCount = config.getMaxBreadcrumbCount(); + currentVZoneTimerInterval = config.content.zoneTimerInterval; + } + + /** + * Loads the stored settings, or the provided ones when nothing is stored yet, which stores them. + * + * @param sdkBehaviorSettings the settings the developer provided, {@code null} for none + */ + void loadConfigFromStorage(@Nullable String sdkBehaviorSettings) { + String sConfig = internalConfig.storageProvider.getServerConfig(); + + if (Utils.isEmptyOrNull(sConfig) && !Utils.isEmptyOrNull(sdkBehaviorSettings)) { + L.d("[ModuleConfiguration] loadConfigFromStorage, nothing is stored, using the provided settings"); + sConfig = sdkBehaviorSettings; + } + + L.v("[ModuleConfiguration] loadConfigFromStorage, [" + sConfig + "]"); + + if (Utils.isEmptyOrNull(sConfig)) { + L.d("[ModuleConfiguration] loadConfigFromStorage, no configuration is stored or provided"); + return; + } + + try { + saveAndStoreDownloadedConfig(new JSONObject(sConfig)); + } catch (JSONException e) { + L.w("[ModuleConfiguration] loadConfigFromStorage, failed to parse, " + e); + } + } + + /** + * Requires {@code v}, {@code t} and an object {@code c}, then removes the settings this SDK does + * not support. Other top level keys are allowed; they are never stored. + * + * @param config a configuration as the server sends it, its settings object is changed in place + * @return whether the configuration can be merged + */ + boolean validateServerConfig(@Nonnull JSONObject config) { + L.v("[ModuleConfiguration] validateServerConfig"); + if (!config.has(keyRVersion)) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has no 'v' field, it is ignored"); + return false; + } + if (!config.has(keyRTimestamp)) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has no 't' field, it is ignored"); + return false; + } + if (!config.has(keyRConfig)) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has no 'c' field, it is ignored"); + return false; + } + + JSONObject newInner = config.optJSONObject(keyRConfig); + if (newInner == null) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has a 'c' that is not an object, it is ignored"); + return false; + } + + removeUnsupportedKeys(newInner); + return true; + } + + /** + * Removes every setting whose key is unknown or whose value has the wrong type or range. + * + * @param newInner the settings object, changed in place + */ + private void removeUnsupportedKeys(@Nonnull JSONObject newInner) { + Iterator keys = newInner.keys(); + while (keys.hasNext()) { + String key = keys.next(); + Object value = newInner.opt(key); + boolean isValid; + + switch (key) { + case keyRNetworking: + case keyRTracking: + case keyRSessionTracking: + case keyRCrashReporting: + case keyRAutomaticSessionTracking: + case keyRAutomaticViewTracking: + case keyRAutomaticCrashReporting: + case keyRViewTracking: + case keyRCustomEventTracking: + case keyRLocationTracking: + case keyREnterContentZone: + case keyRRefreshContentZone: + case keyRBackoffMechanism: + case keyRLogging: + case keyRConsentRequired: + isValid = value instanceof Boolean; + break; + case keyRServerConfigUpdateInterval: + case keyRBOMAcceptedTimeout: + case keyRBOMRequestAge: + case keyRBOMDuration: + case keyRReqQueueSize: + case keyREventQueueSize: + case keyRSessionUpdateInterval: + case keyRLimitKeyLength: + case keyRLimitValueSize: + case keyRLimitSegValues: + case keyRLimitBreadcrumb: + case keyRLimitTraceLine: + case keyRLimitTraceLength: + case keyRUserPropertyCacheLimit: + isValid = isIntAtLeast(value, 1); + break; + case keyRDropOldRequestTime: + isValid = isIntAtLeast(value, 0); + break; + case keyRContentZoneInterval: + isValid = isIntAtLeast(value, MIN_CONTENT_ZONE_INTERVAL); + break; + case keyRBOMRQPercentage: + isValid = isFraction(value); + break; + case keyREventBlacklist: + case keyRSegmentationBlacklist: + case keyRUserPropertyBlacklist: + case keyREventWhitelist: + case keyRSegmentationWhitelist: + case keyRUserPropertyWhitelist: + case keyRJourneyTriggerEvents: + case keyRJourneyTriggerViews: + isValid = value instanceof JSONArray; + break; + case keyREventSegmentationBlacklist: + case keyREventSegmentationWhitelist: + isValid = value instanceof JSONObject; + break; + default: + L.w("[ModuleConfiguration] removeUnsupportedKeys, unknown key [" + key + "], removing it, value: [" + value + "]"); + keys.remove(); + continue; + } + + if (!isValid) { + L.w("[ModuleConfiguration] removeUnsupportedKeys, invalid value for key [" + key + "], removing it, value: [" + value + "]"); + keys.remove(); + } + } + } + + /** + * Whether a setting value is an integer of at least the given value. + * + * @param value the value as parsed from JSON + * @param minimum the smallest valid value + * @return {@code true} for a valid value + */ + private static boolean isIntAtLeast(@Nullable Object value, int minimum) { + return value instanceof Integer && (Integer) value >= minimum; + } + + /** + * Whether a setting value is a number strictly between 0 and 1. Decimals are parsed as + * {@link java.math.BigDecimal}, so any {@link Number} is accepted. + * + * @param value the value as parsed from JSON + * @return {@code true} for a valid value + */ + private static boolean isFraction(@Nullable Object value) { + if (!(value instanceof Number)) { + return false; + } + double fraction = ((Number) value).doubleValue(); + return fraction > 0.0 && fraction < 1.0; + } + + /** + * Validates a configuration, merges its settings into the stored ones and stores the result. + * Only {@code v}, {@code t} and {@code c} are stored, so the log gathering directive and the + * connection test flag never are. An invalid configuration leaves the stored settings as they are. + * + * @param config a configuration as the server sends it + */ + synchronized void saveAndStoreDownloadedConfig(@Nonnull JSONObject config) { + L.v("[ModuleConfiguration] saveAndStoreDownloadedConfig"); + if (!validateServerConfig(config)) { + L.w("[ModuleConfiguration] saveAndStoreDownloadedConfig, retrieved configuration is not valid, ignoring it"); + return; + } + + JSONObject newInner = config.optJSONObject(keyRConfig); + if (latestRetrievedConfigurationFull == null) { + latestRetrievedConfiguration = new JSONObject(); + latestRetrievedConfigurationFull = new JSONObject(); + latestRetrievedConfigurationFull.put(keyRConfig, latestRetrievedConfiguration); + } + + latestRetrievedConfigurationFull.put(keyRVersion, config.get(keyRVersion)); + latestRetrievedConfigurationFull.put(keyRTimestamp, config.get(keyRTimestamp)); + + List notStoredTopLevelKeys = new ArrayList<>(); + for (String key : config.keySet()) { + if (!keyRVersion.equals(key) && !keyRTimestamp.equals(key) && !keyRConfig.equals(key)) { + notStoredTopLevelKeys.add(key); + } + } + if (!notStoredTopLevelKeys.isEmpty()) { + L.d("[ModuleConfiguration] saveAndStoreDownloadedConfig, top level keys that are not stored: " + notStoredTopLevelKeys); + } + + removeListingFilterKeysFromConfig(newInner); + + for (String key : newInner.keySet()) { + Object value = newInner.opt(key); + if (value != null && !JSONObject.NULL.equals(value)) { + latestRetrievedConfiguration.put(key, value); + } + } + + internalConfig.storageProvider.setServerConfig(latestRetrievedConfigurationFull.toString()); + } + + /** + * Drops the stored listing filters of the opposite kind: a configuration with any whitelist + * removes every stored blacklist and the reverse. One with neither keeps the stored filters. + * + * @param newConfig the settings object being merged + */ + private void removeListingFilterKeysFromConfig(@Nonnull JSONObject newConfig) { + boolean hasAnyWhitelist = newConfig.has(keyREventWhitelist) + || newConfig.has(keyRUserPropertyWhitelist) + || newConfig.has(keyRSegmentationWhitelist) + || newConfig.has(keyREventSegmentationWhitelist); + + boolean hasAnyBlacklist = newConfig.has(keyREventBlacklist) + || newConfig.has(keyRUserPropertyBlacklist) + || newConfig.has(keyRSegmentationBlacklist) + || newConfig.has(keyREventSegmentationBlacklist); + + if (hasAnyWhitelist) { + latestRetrievedConfiguration.remove(keyREventBlacklist); + latestRetrievedConfiguration.remove(keyRUserPropertyBlacklist); + latestRetrievedConfiguration.remove(keyRSegmentationBlacklist); + latestRetrievedConfiguration.remove(keyREventSegmentationBlacklist); + } + + if (hasAnyBlacklist) { + latestRetrievedConfiguration.remove(keyREventWhitelist); + latestRetrievedConfiguration.remove(keyRUserPropertyWhitelist); + latestRetrievedConfiguration.remove(keyRSegmentationWhitelist); + latestRetrievedConfiguration.remove(keyREventSegmentationWhitelist); + } + } + + /** + * Applies the merged settings to the resolved values, decides the log gathering directive of a + * live response and restarts the refresh timer when its interval changed. + * + * @param config configuration of the running SDK + * @param serverResponse the live response being applied, {@code null} at init, where only stored or provided settings exist + * @return whether a resolved value changed + */ + synchronized boolean updateConfigVariables(@Nonnull InternalConfig config, @Nullable JSONObject serverResponse) { + L.v("[ModuleConfiguration] updateConfigVariables, from server response:[" + (serverResponse != null) + "]"); + + //read off the live response itself, so it applies even when its settings object was rejected + readLogGatheringDirective(serverResponse); + + if (latestRetrievedConfiguration == null) { + return false; + } + + StringBuilder sb = new StringBuilder(); + boolean previousRequiresConsent = currentVRequiresConsent; + + currentVNetworking = extractBoolean(keyRNetworking, sb, currentVNetworking); + currentVTracking = extractBoolean(keyRTracking, sb, currentVTracking); + currentVSessionTracking = extractBoolean(keyRSessionTracking, sb, currentVSessionTracking); + currentVCrashReporting = extractBoolean(keyRCrashReporting, sb, currentVCrashReporting); + currentVAutomaticSessionTracking = extractBoolean(keyRAutomaticSessionTracking, sb, currentVAutomaticSessionTracking); + currentVAutomaticViewTracking = extractBoolean(keyRAutomaticViewTracking, sb, currentVAutomaticViewTracking); + currentVAutomaticCrashReporting = extractBoolean(keyRAutomaticCrashReporting, sb, currentVAutomaticCrashReporting); + currentVViewTracking = extractBoolean(keyRViewTracking, sb, currentVViewTracking); + currentVCustomEventTracking = extractBoolean(keyRCustomEventTracking, sb, currentVCustomEventTracking); + currentVLocationTracking = extractBoolean(keyRLocationTracking, sb, currentVLocationTracking); + currentVContentZone = extractBoolean(keyREnterContentZone, sb, currentVContentZone); + currentVRefreshContentZone = extractBoolean(keyRRefreshContentZone, sb, currentVRefreshContentZone); + currentVBackoffMechanism = extractBoolean(keyRBackoffMechanism, sb, currentVBackoffMechanism); + currentVLoggingEnabled = extractBoolean(keyRLogging, sb, currentVLoggingEnabled); + currentVRequiresConsent = extractBoolean(keyRConsentRequired, sb, currentVRequiresConsent); + + currentVServerConfigUpdateInterval = extractInt(keyRServerConfigUpdateInterval, sb, currentVServerConfigUpdateInterval, 1); + currentVRequestQueueMaxSize = extractInt(keyRReqQueueSize, sb, currentVRequestQueueMaxSize, 1); + currentVEventQueueSizeThreshold = extractInt(keyREventQueueSize, sb, currentVEventQueueSizeThreshold, 1); + currentVSessionUpdateInterval = extractInt(keyRSessionUpdateInterval, sb, currentVSessionUpdateInterval, 1); + currentVMaxKeyLength = extractInt(keyRLimitKeyLength, sb, currentVMaxKeyLength, 1); + currentVMaxValueSize = extractInt(keyRLimitValueSize, sb, currentVMaxValueSize, 1); + currentVMaxSegmentationValues = extractInt(keyRLimitSegValues, sb, currentVMaxSegmentationValues, 1); + currentVMaxBreadcrumbCount = extractInt(keyRLimitBreadcrumb, sb, currentVMaxBreadcrumbCount, 1); + currentVMaxStackTraceLinesPerThread = extractInt(keyRLimitTraceLine, sb, currentVMaxStackTraceLinesPerThread, 1); + currentVMaxStackTraceLineLength = extractInt(keyRLimitTraceLength, sb, currentVMaxStackTraceLineLength, 1); + currentVUserPropertyCacheLimit = extractInt(keyRUserPropertyCacheLimit, sb, currentVUserPropertyCacheLimit, 1); + currentVBOMAcceptedTimeoutSeconds = extractInt(keyRBOMAcceptedTimeout, sb, currentVBOMAcceptedTimeoutSeconds, 1); + currentVBOMRQPercentage = extractFraction(keyRBOMRQPercentage, sb, currentVBOMRQPercentage); + currentVBOMRequestAge = extractInt(keyRBOMRequestAge, sb, currentVBOMRequestAge, 1); + currentVBOMDuration = extractInt(keyRBOMDuration, sb, currentVBOMDuration, 1); + currentVDropAgeHours = extractInt(keyRDropOldRequestTime, sb, currentVDropAgeHours, 0); + currentVZoneTimerInterval = extractInt(keyRContentZoneInterval, sb, currentVZoneTimerInterval, MIN_CONTENT_ZONE_INTERVAL); + currentVRequestQueueMaxSizeFromBehaviorSettings = latestRetrievedConfiguration.has(keyRReqQueueSize); + + updateListingFilters(); + + if (serverResponse != null && previousRequiresConsent != currentVRequiresConsent) { + L.d("[ModuleConfiguration] updateConfigVariables, consent requirement changed to [" + currentVRequiresConsent + "], it applies on the next init, the running SDK keeps [" + config.requiresConsent() + "]"); + } + + if (serverConfigUpdateTimer != null && serverConfigUpdateTimer.getTimerDelaySeconds() != serverConfigUpdateIntervalSeconds()) { + startServerConfigUpdateTimer(); + } + + String updatedValues = sb.toString(); + if (updatedValues.isEmpty()) { + return false; + } + + L.i("[ModuleConfiguration] updateConfigVariables, SDK configuration has changed, new values: [" + updatedValues + "]"); + return true; + } + + /** + * Reads a switch from the merged settings. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @return the stored value, or the value in effect when none is stored + */ + private boolean extractBoolean(@Nonnull String key, @Nonnull StringBuilder changedValues, boolean currentValue) { + Object value = latestRetrievedConfiguration.opt(key); + if (!(value instanceof Boolean)) { + return currentValue; + } + + boolean extracted = (Boolean) value; + if (extracted != currentValue) { + changedValues.append(key).append(":[").append(extracted).append("], "); + } + return extracted; + } + + /** + * Reads an integer from the merged settings. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @param minimum the smallest valid value + * @return the stored value, or the value in effect when none, or none valid, is stored + */ + private int extractInt(@Nonnull String key, @Nonnull StringBuilder changedValues, int currentValue, int minimum) { + Object value = latestRetrievedConfiguration.opt(key); + if (!(value instanceof Integer)) { + return currentValue; + } + + int extracted = (Integer) value; + if (extracted < minimum) { + L.w("[ModuleConfiguration] updateConfigVariables, value for '" + key + "' is not valid, value: [" + extracted + "]"); + return currentValue; + } + + if (extracted != currentValue) { + changedValues.append(key).append(":[").append(extracted).append("], "); + } + return extracted; + } + + /** + * Reads a fraction strictly between 0 and 1 from the merged settings. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @return the stored value, or the value in effect when none, or none valid, is stored + */ + private double extractFraction(@Nonnull String key, @Nonnull StringBuilder changedValues, double currentValue) { + Object value = latestRetrievedConfiguration.opt(key); + if (!(value instanceof Number)) { + return currentValue; + } + + double extracted = ((Number) value).doubleValue(); + if (!isFraction(value)) { + L.w("[ModuleConfiguration] updateConfigVariables, value for '" + key + "' is not valid, value: [" + extracted + "]"); + return currentValue; + } + + if (Double.compare(extracted, currentValue) != 0) { + changedValues.append(key).append(":[").append(extracted).append("], "); + } + return extracted; + } + + /** + * Rebuilds the listing filters and the journey trigger sets from the merged settings and + * publishes each as a new immutable snapshot. A filter the merged settings do not hold is an + * empty blacklist, which allows everything. + */ + private void updateListingFilters() { + currentVEventFilterList = namesFilterOf(keyREventBlacklist, keyREventWhitelist); + currentVUserPropertyFilterList = namesFilterOf(keyRUserPropertyBlacklist, keyRUserPropertyWhitelist); + currentVSegmentationFilterList = namesFilterOf(keyRSegmentationBlacklist, keyRSegmentationWhitelist); + currentVEventSegmentationFilterList = namesPerEventFilterOf(keyREventSegmentationBlacklist, keyREventSegmentationWhitelist); + currentVJourneyTriggerEvents = Collections.unmodifiableSet(namesOf(latestRetrievedConfiguration.optJSONArray(keyRJourneyTriggerEvents))); + currentVJourneyTriggerViews = Collections.unmodifiableSet(namesOf(latestRetrievedConfiguration.optJSONArray(keyRJourneyTriggerViews))); + + L.d("[ModuleConfiguration] updateListingFilters, event filter:[" + currentVEventFilterList + + "], user property filter:[" + currentVUserPropertyFilterList + + "], segmentation filter:[" + currentVSegmentationFilterList + + "], event segmentation filter:[" + currentVEventSegmentationFilterList + + "], journey trigger events:" + currentVJourneyTriggerEvents + + ", journey trigger views:" + currentVJourneyTriggerViews); + } + + /** + * Builds a names filter from the merged settings, the blacklist winning when both are stored. + * + * @param blacklistKey the settings key of the blacklist + * @param whitelistKey the settings key of the whitelist + * @return the filter + */ + @Nonnull + private FilterList> namesFilterOf(@Nonnull String blacklistKey, @Nonnull String whitelistKey) { + JSONArray blacklist = latestRetrievedConfiguration.optJSONArray(blacklistKey); + if (blacklist != null) { + return FilterList.ofNames(namesOf(blacklist), false); + } + + JSONArray whitelist = latestRetrievedConfiguration.optJSONArray(whitelistKey); + if (whitelist != null) { + return FilterList.ofNames(namesOf(whitelist), true); + } + + return FilterList.NO_NAMES; + } + + /** + * Builds a names per event filter from the merged settings, the blacklist winning when both are stored. + * + * @param blacklistKey the settings key of the blacklist + * @param whitelistKey the settings key of the whitelist + * @return the filter + */ + @Nonnull + private FilterList>> namesPerEventFilterOf(@Nonnull String blacklistKey, @Nonnull String whitelistKey) { + JSONObject blacklist = latestRetrievedConfiguration.optJSONObject(blacklistKey); + if (blacklist != null) { + return FilterList.ofNamesPerEvent(namesPerEventOf(blacklist), false); + } + + JSONObject whitelist = latestRetrievedConfiguration.optJSONObject(whitelistKey); + if (whitelist != null) { + return FilterList.ofNamesPerEvent(namesPerEventOf(whitelist), true); + } + + return FilterList.NO_NAMES_PER_EVENT; + } + + /** + * Reads the names of a JSON array, skipping nulls and turning other values into strings. + * + * @param jsonArray the array, {@code null} for none + * @return the names in array order + */ + @Nonnull + private static Set namesOf(@Nullable JSONArray jsonArray) { + Set names = new LinkedHashSet<>(); + if (jsonArray == null) { + return names; + } + + for (int i = 0; i < jsonArray.length(); i++) { + String item = jsonArray.optString(i, null); + if (item != null) { + names.add(item); + } + } + return names; + } + + /** + * Reads the names of each event key of a JSON object, skipping keys whose value is not an array. + * + * @param jsonObject the object of arrays + * @return the names of each event key + */ + @Nonnull + private static Map> namesPerEventOf(@Nonnull JSONObject jsonObject) { + Map> namesPerEvent = new LinkedHashMap<>(); + for (String key : jsonObject.keySet()) { + JSONArray jsonArray = jsonObject.optJSONArray(key); + if (jsonArray != null) { + namesPerEvent.put(key, namesOf(jsonArray)); + } + } + return namesPerEvent; + } + + /** + * Decides the log gathering state from the top level {@code lg} directive, shaped + * {@code {"e":false}} or {@code {"e":true,"i":id,"l":"ewidv","b":100}}. Only a live response + * decides: without one the state stays as it is, while disabled settings requests, a response + * without a usable directive and a directive with {@code e} not {@code true} decide off. + * + * @param serverResponse the live response, {@code null} at init + */ + private synchronized void readLogGatheringDirective(@Nullable JSONObject serverResponse) { + if (serverConfigRequestsDisabled) { + //without settings requests no directive can ever arrive, so nothing can ever be gathered + L.d("[ModuleConfiguration] readLogGatheringDirective, SDK behavior settings requests are disabled, log gathering can never be armed"); + setLogGatheringOff(); + return; + } + + if (serverResponse == null) { + L.d("[ModuleConfiguration] readLogGatheringDirective, not a server response, log gathering stays undecided"); + return; + } + + JSONObject directive = serverResponse.optJSONObject(keyRLogGathering); + if (directive == null) { + L.d("[ModuleConfiguration] readLogGatheringDirective, server response had no usable '" + keyRLogGathering + "', log gathering is off"); + setLogGatheringOff(); + return; + } + + if (!Boolean.TRUE.equals(directive.opt(keyLGEnabled))) { + L.d("[ModuleConfiguration] readLogGatheringDirective, directive says log gathering is off"); + setLogGatheringOff(); + return; + } + + Object gatherIdRaw = directive.opt(keyLGId); + String gatherId = gatherIdRaw instanceof String ? ((String) gatherIdRaw).trim() : ""; + if (gatherId.isEmpty()) { + //without the id the server rejects every uploaded batch + L.d("[ModuleConfiguration] readLogGatheringDirective, directive enables log gathering but carries no usable '" + keyLGId + "', log gathering is off"); + setLogGatheringOff(); + return; + } + + LogGatheringDirective gathering = new LogGatheringDirective(LogGatheringState.GATHERING, gatherId, + sanitizeLogGatheringLevels(directive.opt(keyLGLevels)), sanitizeLogGatheringBatchSize(directive.opt(keyLGBatchSize))); + currentVLogGathering = gathering; + L.d("[ModuleConfiguration] readLogGatheringDirective, log gathering is on, id:[" + gathering.gatherId + "], levels:[" + gathering.levels + "], batch size:[" + gathering.batchSize + "]"); + } + + /** + * Decides against log gathering when nothing decided yet, for a run where no response can arrive. + * + * @param reason why no response can arrive + */ + private synchronized void decideLogGatheringOffIfUndecided(@Nonnull String reason) { + if (currentVLogGathering.state == LogGatheringState.UNDECIDED) { + L.d("[ModuleConfiguration] decideLogGatheringOffIfUndecided, " + reason + ", log gathering is off"); + setLogGatheringOff(); + } + } + + /** + * Publishes the decision against log gathering, which also drops any previous gather id. + */ + private void setLogGatheringOff() { + currentVLogGathering = LogGatheringDirective.NOT_GATHERING; + } + + /** + * Keeps only the level characters this SDK knows, lower cased, in the order the server sent them + * and without duplicates. Falls back to every level when nothing usable is left. + * + * @param levelsRaw the {@code l} value of the directive + * @return the level characters to gather + */ + @Nonnull + private String sanitizeLogGatheringLevels(@Nullable Object levelsRaw) { + if (!(levelsRaw instanceof String)) { + return logGatheringAllLevels; + } + + String levels = (String) levelsRaw; + StringBuilder filtered = new StringBuilder(); + for (int i = 0; i < levels.length(); i++) { + char level = Character.toLowerCase(levels.charAt(i)); + if (logGatheringAllLevels.indexOf(level) > -1 && filtered.indexOf(String.valueOf(level)) < 0) { + filtered.append(level); + } + } + + if (filtered.length() == 0) { + L.d("[ModuleConfiguration] sanitizeLogGatheringLevels, no usable level in [" + levels + "], falling back to [" + logGatheringAllLevels + "]"); + return logGatheringAllLevels; + } + + return filtered.toString(); + } + + /** + * Clamps the batch size into [{@value #logGatheringMinBatchSize}, {@value #logGatheringMaxBufferedLines}], + * falling back to {@value #logGatheringDefaultBatchSize} when it is missing or not a number. + * + * @param batchSizeRaw the {@code b} value of the directive + * @return the batch size + */ + private static int sanitizeLogGatheringBatchSize(@Nullable Object batchSizeRaw) { + if (!(batchSizeRaw instanceof Number)) { + return logGatheringDefaultBatchSize; + } + + int batchSize = ((Number) batchSizeRaw).intValue(); + return Math.min(logGatheringMaxBufferedLines, Math.max(logGatheringMinBatchSize, batchSize)); + } + + /** + * Requests the settings from the server and applies the response. Skipped when settings + * requests are disabled, in backend mode and while there is no device ID. The request is sent + * even while the settings forbid networking, as the next response is the only way to allow it again. + * + * @param config configuration of the running SDK + */ + void fetchConfigFromServer(@Nonnull InternalConfig config) { + L.v("[ModuleConfiguration] fetchConfigFromServer"); + if (!running || serverConfigRequestsDisabled) { + L.v("[ModuleConfiguration] fetchConfigFromServer, fetching the configuration is aborted, server config requests are disabled or the SDK is stopped"); + return; + } + + if (config.getDeviceId() == null) { + L.d("[ModuleConfiguration] fetchConfigFromServer, fetching the configuration is aborted, there is no device ID yet"); + decideLogGatheringOffIfUndecided("no device ID, no server response this run"); + return; + } + + if (config.sdk == null || config.sdk.networking == null) { + L.w("[ModuleConfiguration] fetchConfigFromServer, fetching the configuration is aborted, networking is not available"); + decideLogGatheringOffIfUndecided("networking is not available, no server response this run"); + return; + } + + try { + String requestData = ModuleRequests.prepareRequiredParams(config).add("method", "sc").toString(); + Transport transport = config.sdk.networking.getTransport(); + final long fetchStartNs = System.nanoTime(); + + config.immediateRequestGenerator.createImmediateRequestMaker().doWork(requestData, "/o/sdk?", transport, false, true, + response -> onServerConfigResponse(config, response, fetchStartNs), L); + } catch (Exception e) { + L.e("[ModuleConfiguration] fetchConfigFromServer, failed to request the configuration, [" + e + "]"); + decideLogGatheringOffIfUndecided("the server config request failed"); + } + } + + /** + * Handles the response of a settings request: strips the connection test flag, merges, stores + * and applies the settings, and tells the SDK when a resolved value changed. + * + * @param config configuration of the running SDK + * @param response the response, {@code null} when the request failed + * @param fetchStartNs {@link System#nanoTime()} just before the request was made + */ + private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JSONObject response, long fetchStartNs) { + if (!running) { + L.d("[ModuleConfiguration] onServerConfigResponse, the SDK was stopped before the response arrived, ignoring it"); + return; + } + + if (response == null) { + L.w("[ModuleConfiguration] onServerConfigResponse, not possible to retrieve configuration data, probably due to lack of connection to the server"); + decideLogGatheringOffIfUndecided("server config fetch failed"); + return; + } + + L.d("[ModuleConfiguration] onServerConfigResponse, retrieved configuration response: [" + response + "]"); + long fetchLatencyMs = (System.nanoTime() - fetchStartNs) / 1_000_000L; + boolean connectionTestRequested = extractConnectionTestFlag(response); + + boolean changed; + synchronized (this) { + //checked again under the lock stop() takes, so a response racing stop() never writes the store + if (!running) { + L.d("[ModuleConfiguration] onServerConfigResponse, the SDK was stopped while the response was read, ignoring it"); + return; + } + saveAndStoreDownloadedConfig(response); + changed = updateConfigVariables(config, response); + } + + if (changed && config.sdk != null) { + config.sdk.onSdkConfigurationChanged(config); + } + + if (connectionTestRequested) { + notifyConnectionTestRequested(fetchLatencyMs); + } + } + + /** + * Reads and removes the {@code ct} flag of a live response, so it is never stored and can never + * come back from storage. Booleans and numbers count by value; a string counts unless it is + * empty, {@code "0"} or {@code "false"}; any other value that is not null counts. + * + * @param serverConfigResponse the live response, changed in place + * @return whether the response asks for a connection test + */ + static boolean extractConnectionTestFlag(@Nullable JSONObject serverConfigResponse) { + if (serverConfigResponse == null || !serverConfigResponse.has(keyRConnectionTest)) { + return false; + } + + Object value = serverConfigResponse.remove(keyRConnectionTest); + if (value == null || JSONObject.NULL.equals(value)) { + return false; + } + if (value instanceof Boolean) { + return (Boolean) value; + } + if (value instanceof Number) { + return ((Number) value).doubleValue() != 0; + } + if (value instanceof String) { + String s = ((String) value).trim(); + return !s.isEmpty() && !s.equals("0") && !s.equalsIgnoreCase("false"); + } + return true; + } + + /** + * Hands a requested connection test to the registered listener. + * + * @param fetchLatencyMs how long the settings request took, in milliseconds + */ + private void notifyConnectionTestRequested(long fetchLatencyMs) { + ConnectionTestListener listener = connectionTestListener; + if (listener == null) { + L.d("[ModuleConfiguration] notifyConnectionTestRequested, the server asked for a connection test, nothing is registered to run it"); + return; + } + + try { + listener.onConnectionTestRequested(fetchLatencyMs); + } catch (Exception e) { + L.e("[ModuleConfiguration] notifyConnectionTestRequested, the connection test listener failed, [" + e + "]"); + } + } + + /** + * Registers what runs the connection test a live settings response can ask for. + * + * @param listener called with the latency of the settings request, {@code null} to unregister + */ + void setConnectionTestListener(@Nullable ConnectionTestListener listener) { + connectionTestListener = listener; + } + + /** + * The refresh interval in the unit of the timer. + * + * @return seconds between two settings fetches + */ + private long serverConfigUpdateIntervalSeconds() { + return (long) currentVServerConfigUpdateInterval * 60L * 60L; + } + + /** + * Starts the timer that fetches the settings every {@code scui} hours, replacing a running one. + * Never waits for the replaced timer, as this also runs from that timer's own task. + */ + private synchronized void startServerConfigUpdateTimer() { + if (serverConfigUpdateTimer != null) { + serverConfigUpdateTimer.stopTimer(false); + } + + L.d("[ModuleConfiguration] startServerConfigUpdateTimer, fetching the configuration every [" + currentVServerConfigUpdateInterval + "] hours"); + final InternalConfig config = internalConfig; + serverConfigUpdateTimer = new CountlyTimer(L); + serverConfigUpdateTimer.startTimer(serverConfigUpdateIntervalSeconds(), () -> fetchConfigFromServer(config)); + } + + /** + * Stops the refresh timer, if it runs. + */ + private synchronized void stopServerConfigUpdateTimer() { + if (serverConfigUpdateTimer != null) { + serverConfigUpdateTimer.stopTimer(false); + serverConfigUpdateTimer = null; + } + } + + /** {@inheritDoc} */ + @Override + public boolean getNetworkingEnabled() { + return currentVNetworking; + } + + /** {@inheritDoc} */ + @Override + public boolean getTrackingEnabled() { + return currentVTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getSessionTrackingEnabled() { + return currentVSessionTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getViewTrackingEnabled() { + return currentVViewTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getCustomEventTrackingEnabled() { + return currentVCustomEventTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getContentZoneEnabled() { + return currentVContentZone; + } + + /** {@inheritDoc} */ + @Override + public boolean getCrashReportingEnabled() { + return currentVCrashReporting; + } + + /** {@inheritDoc} */ + @Override + public boolean getAutomaticSessionTrackingEnabled() { + return currentVAutomaticSessionTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getAutomaticViewTrackingEnabled() { + return currentVAutomaticViewTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getAutomaticCrashReportingEnabled() { + return currentVAutomaticCrashReporting; + } + + /** {@inheritDoc} */ + @Override + public boolean getLocationTrackingEnabled() { + return currentVLocationTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getRefreshContentZoneEnabled() { + return currentVRefreshContentZone; + } + + /** {@inheritDoc} */ + @Override + public boolean getBOMEnabled() { + return currentVBackoffMechanism; + } + + /** {@inheritDoc} */ + @Override + public boolean getLoggingEnabled() { + return currentVLoggingEnabled; + } + + /** {@inheritDoc} */ + @Override + public boolean getConsentRequired() { + return currentVRequiresConsent; + } + + /** {@inheritDoc} */ + @Override + public int getServerConfigUpdateInterval() { + return currentVServerConfigUpdateInterval; + } + + /** {@inheritDoc} */ + @Override + public int getRequestQueueMaxSize() { + return currentVRequestQueueMaxSize; + } + + /** {@inheritDoc} */ + @Override + public boolean isRequestQueueMaxSizeFromBehaviorSettings() { + return currentVRequestQueueMaxSizeFromBehaviorSettings; + } + + /** {@inheritDoc} */ + @Override + public int getEventQueueSizeThreshold() { + return currentVEventQueueSizeThreshold; + } + + /** {@inheritDoc} */ + @Override + public int getSessionUpdateInterval() { + return currentVSessionUpdateInterval; + } + + /** {@inheritDoc} */ + @Override + public int getMaxKeyLength() { + return currentVMaxKeyLength; + } + + /** {@inheritDoc} */ + @Override + public int getMaxValueSize() { + return currentVMaxValueSize; + } + + /** {@inheritDoc} */ + @Override + public int getMaxSegmentationValues() { + return currentVMaxSegmentationValues; + } + + /** {@inheritDoc} */ + @Override + public int getMaxBreadcrumbCount() { + return currentVMaxBreadcrumbCount; + } + + /** {@inheritDoc} */ + @Override + public int getMaxStackTraceLinesPerThread() { + return currentVMaxStackTraceLinesPerThread; + } + + /** {@inheritDoc} */ + @Override + public int getMaxStackTraceLineLength() { + return currentVMaxStackTraceLineLength; + } + + /** {@inheritDoc} */ + @Override + public int getUserPropertyCacheLimit() { + return currentVUserPropertyCacheLimit; + } + + /** {@inheritDoc} */ + @Override + public int getBOMAcceptedTimeoutSeconds() { + return currentVBOMAcceptedTimeoutSeconds; + } + + /** {@inheritDoc} */ + @Override + public double getBOMRQPercentage() { + return currentVBOMRQPercentage; + } + + /** {@inheritDoc} */ + @Override + public int getBOMRequestAge() { + return currentVBOMRequestAge; + } + + /** {@inheritDoc} */ + @Override + public int getBOMDuration() { + return currentVBOMDuration; + } + + /** {@inheritDoc} */ + @Override + public int getRequestDropAgeHours() { + return currentVDropAgeHours; + } + + /** {@inheritDoc} */ + @Override + public int getContentZoneTimerInterval() { + return currentVZoneTimerInterval; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList> getEventFilterList() { + return currentVEventFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList> getUserPropertyFilterList() { + return currentVUserPropertyFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList> getSegmentationFilterList() { + return currentVSegmentationFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList>> getEventSegmentationFilterList() { + return currentVEventSegmentationFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public Set getJourneyTriggerEvents() { + return currentVJourneyTriggerEvents; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public Set getJourneyTriggerViews() { + return currentVJourneyTriggerViews; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public LogGatheringState getLogGatheringState() { + return currentVLogGathering.state; + } + + /** {@inheritDoc} */ + @Nullable + @Override + public String getLogGatheringId() { + return currentVLogGathering.gatherId; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public String getLogGatheringLevels() { + return currentVLogGathering.levels; + } + + /** {@inheritDoc} */ + @Override + public int getLogGatheringBatchSize() { + return currentVLogGathering.batchSize; + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConnectionTest.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConnectionTest.java new file mode 100644 index 00000000..83069332 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConnectionTest.java @@ -0,0 +1,603 @@ +package ly.count.sdk.java.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Runs the connection test a live SDK behavior settings response asks for with its top level + * {@code ct} flag: one battery of parameterless GET probes against every endpoint the SDK depends + * on, graded row by row and queued as one {@code ct_results} report. The feature keys, their order + * and the {@code e} strings are the ones every Countly SDK reports, so the dashboard can group the + * results. + *

+ * The module is built for every SDK, whatever its features and consent, and listens to + * {@link ModuleConfiguration}, which reads the flag. One battery runs at a time, on a daemon thread + * of its own, and none while the settings switch networking off. Probes go straight to the server, + * past the request queue, so neither the backoff mechanism nor the request drop age can hold them + * back, while the report is an ordinary queued request. Nothing about the test is ever stored, so a + * response that asks again runs the battery again. + */ +class ModuleConnectionTest extends ModuleBase implements ModuleConfiguration.ConnectionTestListener { + static final String keyResults = "ct_results"; + static final String THREAD_NAME = "Countly-ConnectionTest"; + + static final int DEFAULT_PER_REQUEST_TIMEOUT_MS = 10_000; + static final long BATTERY_CAP_EXTRA_MS = 30_000; + static final int MAX_ROWS = 32; + static final int MAX_REPORT_BYTES = 8 * 1024; + static final int MAX_ERROR_LENGTH = 256; + + static final String errorTimeout = "timeout"; + static final String errorTransport = "transport error"; + static final String errorRedirected = "redirected"; + static final String errorNotRun = "not run: battery cap"; + //the qualifier of a success graded from an opaque browser probe: never produced here, but it always survives the size cap + static final String errorOpaque = "opaque"; + + /** + * The feature rows in report order. The row without paths is {@code sc}, measured from the + * settings request that asked for the test. + */ + static final ProbeRow[] ROWS = { + new ProbeRow("core", true, "/o/ping"), + new ProbeRow("core-write", false, "/i"), + new ProbeRow("sc", false), + new ProbeRow("rc", false, "/o/sdk?method=rc"), + new ProbeRow("ab", false, "/o/sdk?method=ab_fetch_variants"), + new ProbeRow("feedback", false, "/o/sdk?method=feedback"), + new ProbeRow("feedback-widget", false, "/o/surveys/nps/widget", "/o/surveys/survey/widget", "/o/feedback/widget"), + new ProbeRow("feedback-submit", false, "/i/feedback/inputs"), + new ProbeRow("content", false, "/o/sdk/content"), + new ProbeRow("feedback-page", false, "/feedback/nps", "/feedback/survey", "/feedback/rating"), + new ProbeRow("feedback-assets", true, "/surveys/images/ct-probe.png", "/star-rating/images/ct-probe.png"), + new ProbeRow("content-page", false, "/_external/content/"), + }; + + /** + * One feature row of the battery. + */ + static final class ProbeRow { + final String feature; + /** + * Whether only a 2xx counts as reached: {@code core} and {@code feedback-assets} have no + * legitimate 4xx answer, so anything else is a fault there. + */ + final boolean requiresSuccessStatus; + final String[] paths; + + /** + * Describes one row. + * + * @param feature the feature key, reported as {@code f} + * @param requiresSuccessStatus whether only a 2xx counts as reached + * @param paths the paths to probe, relative to the server URL; none for the {@code sc} row + */ + ProbeRow(@Nonnull String feature, boolean requiresSuccessStatus, @Nonnull String... paths) { + this.feature = feature; + this.requiresSuccessStatus = requiresSuccessStatus; + this.paths = paths; + } + } + + /** + * What one probe observed, before it is graded. + */ + static final class ProbeOutcome { + /** + * The HTTP status, {@code 0} when none could be read. + */ + int status; + /** + * From issuing the request to reading the whole response, or to the failure. + */ + long ms; + /** + * {@link #errorTimeout} or {@link #errorTransport} while the status is {@code 0}, otherwise {@code null}. + */ + String failure; + } + + /** + * One graded row of the report. + */ + static final class ResultRow { + final String feature; + boolean ok; + int status; + long ms; + String error; + int pathCount; + + /** + * Starts a row that reached nothing yet. + * + * @param feature the feature key + */ + ResultRow(@Nonnull String feature) { + this.feature = feature; + } + } + + /** + * Issues one probe. Tests replace it to take the network out. + */ + interface ProbeTransport { + /** + * Sends one bare GET and reads its status. + * + * @param url the probe URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @param transport the transport of the queued requests, whose pinning and custom request + * headers the probe reuses, {@code null} when the SDK runs without one + * @return what the probe observed + */ + @Nonnull + ProbeOutcome probe(@Nonnull String url, int timeoutMs, @Nullable Transport transport); + } + + final AtomicBoolean batteryRunning = new AtomicBoolean(false); + volatile boolean halted = false; + //taken by stop() to set halted and by the battery to check it and queue the report, so a stopped SDK never gets one + private final Object reportLock = new Object(); + private volatile ModuleConfiguration listenedConfiguration = null; + + //package-private so tests can shrink the deadlines and take the network out + volatile int perRequestTimeoutMs = DEFAULT_PER_REQUEST_TIMEOUT_MS; + volatile long batteryCapExtraMs = BATTERY_CAP_EXTRA_MS; + volatile ProbeTransport transport = new HttpProbeTransport(); + + /** + * Starts listening to the configuration module of the SDK for connection test requests. + * + * @param config configuration of the SDK being initialized + */ + @Override + public void init(InternalConfig config) { + super.init(config); + ModuleConfiguration configuration = config.sdk == null ? null : config.sdk.module(ModuleConfiguration.class); + if (configuration == null) { + L.d("[ModuleConnectionTest] init, no configuration module runs, a connection test can never be requested"); + return; + } + + configuration.setConnectionTestListener(this); + listenedConfiguration = configuration; + L.v("[ModuleConnectionTest] init, listening for connection test requests"); + } + + /** + * Stops listening for connection test requests. A running battery ends at its next row and + * never queues its report. + * + * @param config configuration of the SDK being stopped + * @param clear whether the SDK clears its data + */ + @Override + public void stop(InternalConfig config, boolean clear) { + super.stop(config, clear); + synchronized (reportLock) { + halted = true; + } + + ModuleConfiguration configuration = listenedConfiguration; + listenedConfiguration = null; + if (configuration != null) { + configuration.setConnectionTestListener(null); + } + + if (batteryRunning.get()) { + L.d("[ModuleConnectionTest] stop, the running battery ends at its next row and queues no report"); + } + } + + /** + * Runs the battery a live settings response asked for. + * + * @param fetchLatencyMs how long the settings request took, in milliseconds + */ + @Override + public void onConnectionTestRequested(long fetchLatencyMs) { + startBattery(fetchLatencyMs); + } + + /** + * Starts one battery on a daemon thread of its own, unless the SDK stopped, the settings switch + * networking off or a battery is already running. + * + * @param scLatencyMs latency of the settings request that asked for the test, in milliseconds, + * negative when unknown + */ + void startBattery(final long scLatencyMs) { + if (halted) { + L.d("[ModuleConnectionTest] startBattery, the SDK is stopped, ignoring the request"); + return; + } + + if (!internalConfig.getConfigurationProvider().getNetworkingEnabled()) { + L.d("[ModuleConnectionTest] startBattery, networking is disabled by the SDK behavior settings, ignoring the request"); + return; + } + + if (!batteryRunning.compareAndSet(false, true)) { + L.d("[ModuleConnectionTest] startBattery, a battery is already running, ignoring the request"); + return; + } + + L.i("[ModuleConnectionTest] startBattery, the server asked for a connection test, running the probe battery"); + Thread worker = new Thread(() -> { + try { + runBatteryAndReport(scLatencyMs); + } catch (Throwable t) { + L.e("[ModuleConnectionTest] startBattery, the battery failed, [" + t + "]"); + } finally { + batteryRunning.set(false); + } + }, THREAD_NAME); + worker.setDaemon(true); + + try { + worker.start(); + } catch (Throwable t) { + batteryRunning.set(false); + L.e("[ModuleConnectionTest] startBattery, failed to start the battery thread, [" + t + "]"); + } + } + + /** + * Runs the battery and queues its report as a request of its own, unless the SDK stopped + * meanwhile. + * + * @param scLatencyMs latency of the settings request that asked for the test, in milliseconds, + * negative when unknown + */ + void runBatteryAndReport(long scLatencyMs) { + List rows = runBattery(scLatencyMs); + String report = buildReport(rows); + + synchronized (reportLock) { + if (halted) { + L.d("[ModuleConnectionTest] runBatteryAndReport, the SDK stopped during the battery, dropping the report"); + return; + } + + L.d("[ModuleConnectionTest] runBatteryAndReport, queueing the report [" + report + "]"); + Request request = ModuleRequests.nonSessionRequest(internalConfig); + request.params.add(keyResults, report); + ModuleRequests.pushAsync(internalConfig, request); + } + } + + /** + * Probes the rows in report order, one request at a time. A row the battery cap is reached + * before is reported as not run, and a stop ends the battery before the next row. + * + * @param scLatencyMs latency of the settings request that asked for the test, in milliseconds, + * negative when unknown, which leaves the {@code sc} row out + * @return the graded rows in report order + */ + @Nonnull + List runBattery(long scLatencyMs) { + String serverUrl = internalConfig.getServerURL().toString(); + Transport sdkTransport = sdkTransport(); + ProbeTransport probeTransport = transport; + int timeoutMs = perRequestTimeoutMs; + + int requestCount = 0; + for (ProbeRow row : ROWS) { + requestCount += row.paths.length; + } + long capMs = (long) requestCount * timeoutMs + batteryCapExtraMs; + long batteryStartNs = System.nanoTime(); + + List results = new ArrayList<>(ROWS.length); + for (ProbeRow row : ROWS) { + if (halted) { + L.d("[ModuleConnectionTest] runBattery, the SDK stopped, ending the battery before [" + row.feature + "]"); + break; + } + + if (row.paths.length == 0) { + if (scLatencyMs >= 0) { + //the settings request was answered, or no test would have been asked for + ResultRow sc = new ResultRow(row.feature); + sc.ok = true; + sc.status = HttpURLConnection.HTTP_OK; + sc.ms = scLatencyMs; + results.add(sc); + } + continue; + } + + long elapsedMs = (System.nanoTime() - batteryStartNs) / 1_000_000L; + if (elapsedMs > capMs) { + ResultRow notRun = new ResultRow(row.feature); + notRun.error = errorNotRun; + results.add(notRun); + continue; + } + + results.add(probeRow(row, serverUrl, probeTransport, sdkTransport, timeoutMs)); + } + + L.d("[ModuleConnectionTest] runBattery, [" + results.size() + "] rows graded against [" + serverUrl + "]"); + return results; + } + + /** + * Probes every path of a row and combines them into one row: reached only when every path was, + * the status and the reason of the first failing path, or the first status when none failed, the + * latencies summed and the number of paths. + * + * @param row the row + * @param serverUrl the configured server URL + * @param probeTransport what issues the probes + * @param sdkTransport the transport of the queued requests, {@code null} when there is none + * @param timeoutMs the per request timeout, in milliseconds + * @return the graded row + */ + @Nonnull + ResultRow probeRow(@Nonnull ProbeRow row, @Nonnull String serverUrl, @Nonnull ProbeTransport probeTransport, @Nullable Transport sdkTransport, int timeoutMs) { + ResultRow result = new ResultRow(row.feature); + result.ok = true; + result.pathCount = row.paths.length; + + for (int i = 0; i < row.paths.length; i++) { + String url = buildProbeUrl(serverUrl, row.paths[i]); + ProbeOutcome outcome = probeTransport.probe(url, timeoutMs, sdkTransport); + result.ms += outcome.ms; + + String failure = grade(outcome, row.requiresSuccessStatus); + L.v("[ModuleConnectionTest] probeRow, [" + row.feature + "] " + url + " -> status:[" + outcome.status + "] ms:[" + outcome.ms + "] failure:[" + failure + "]"); + + if (i == 0) { + result.status = outcome.status; + } + + if (failure != null && result.ok) { + result.ok = false; + result.status = outcome.status; + result.error = failure; + } + } + + return result; + } + + /** + * Grades one probe by whether the Countly application answered it: a 2xx did, and so did a + * 4xx other than 403 on a row that has a legitimate 4xx answer. No status, a redirect, a 403, a + * 5xx and anything else did not. + * + * @param outcome what the probe observed + * @param requiresSuccessStatus whether only a 2xx counts as reached on this row + * @return {@code null} when the application answered, otherwise the {@code e} reason + */ + @Nullable + static String grade(@Nonnull ProbeOutcome outcome, boolean requiresSuccessStatus) { + int status = outcome.status; + if (status <= 0) { + return outcome.failure != null ? outcome.failure : errorTransport; + } + if (status >= 200 && status < 300) { + return null; + } + if (status >= 300 && status < 400) { + return errorRedirected; + } + if (!requiresSuccessStatus && status >= 400 && status < 500 && status != HttpURLConnection.HTTP_FORBIDDEN) { + return null; + } + return "HTTP " + status; + } + + /** + * Resolves a path against the configured server URL, keeping any path prefix, with exactly one + * {@code /} at the join, then appends the probe marker and a cache buster. + * + * @param serverUrl the configured server URL + * @param path the path, which may carry a query + * @return the probe URL + */ + @Nonnull + static String buildProbeUrl(@Nonnull String serverUrl, @Nonnull String path) { + String base = serverUrl; + while (base.endsWith("/")) { + base = base.substring(0, base.length() - 1); + } + String joined = path.startsWith("/") ? base + path : base + "/" + path; + String separator = joined.contains("?") ? "&" : "?"; + return joined + separator + "ct=1&_=" + System.currentTimeMillis(); + } + + /** + * Builds the report: the device time, the SDK identity and the rows, within the caps of + * {@value #MAX_ROWS} rows and {@value #MAX_REPORT_BYTES} bytes of UTF-8. Over the size cap every + * {@code e} except {@link #errorOpaque} is dropped first, then rows from the end. + * + * @param rows the graded rows in report order + * @return the report as JSON text + */ + @Nonnull + String buildReport(@Nonnull List rows) { + JSONObject report = new JSONObject(); + JSONArray results = new JSONArray(); + try { + report.put("ts", System.currentTimeMillis()); + JSONObject sdk = new JSONObject(); + sdk.put("name", internalConfig.getSdkName()); + sdk.put("version", internalConfig.getSdkVersion()); + report.put("sdk", sdk); + report.put("results", results); + + int rowLimit = Math.min(rows.size(), MAX_ROWS); + for (int i = 0; i < rowLimit; i++) { + results.put(rowToJson(rows.get(i))); + } + + if (utf8Length(report.toString()) > MAX_REPORT_BYTES) { + L.w("[ModuleConnectionTest] buildReport, the report is over the size cap, dropping the error details"); + for (int i = 0; i < results.length(); i++) { + JSONObject row = results.getJSONObject(i); + if (!errorOpaque.equals(row.optString("e"))) { + row.remove("e"); + } + } + } + + while (utf8Length(report.toString()) > MAX_REPORT_BYTES && results.length() > 0) { + L.w("[ModuleConnectionTest] buildReport, the report is still over the size cap, dropping its last row"); + results.remove(results.length() - 1); + } + } catch (JSONException e) { + L.w("[ModuleConnectionTest] buildReport, failed to build the report, [" + e + "]"); + } + + return report.toString(); + } + + /** + * Serializes one row, its {@code e} cut at {@value #MAX_ERROR_LENGTH} characters and its path + * count only when it probed more than one path. + * + * @param row the graded row + * @return the row as the report carries it + */ + @Nonnull + static JSONObject rowToJson(@Nonnull ResultRow row) { + JSONObject json = new JSONObject(); + json.put("f", row.feature); + json.put("ok", row.ok); + json.put("st", row.status); + json.put("ms", row.ms); + if (row.error != null) { + String error = row.error; + if (error.length() > MAX_ERROR_LENGTH) { + error = error.substring(0, MAX_ERROR_LENGTH); + } + json.put("e", error); + } + if (row.pathCount > 1) { + json.put("n", row.pathCount); + } + return json; + } + + /** + * The size of a text in UTF-8. + * + * @param value the text + * @return its length in bytes + */ + static int utf8Length(@Nonnull String value) { + return value.getBytes(StandardCharsets.UTF_8).length; + } + + /** + * The transport the queued requests go through, whose pinning and custom request headers the + * probes reuse. + * + * @return the transport, {@code null} when the SDK runs without its default networking + */ + @Nullable + private Transport sdkTransport() { + SDKCore sdk = internalConfig.sdk; + Networking networking = sdk == null ? null : sdk.networking; + return networking == null ? null : networking.getTransport(); + } + + /** + * Probes with a bare GET opened by {@link Transport#openProbeConnection(String, int)}. + */ + static final class HttpProbeTransport implements ProbeTransport { + //only the status matters, but the body is read so 'ms' covers the whole response, up to this much of it + static final int MAX_DRAIN_BYTES = 64 * 1024; + + /** + * Sends the probe and reads its status and its body, mapping a timeout to + * {@link #errorTimeout} and any other failure to {@link #errorTransport}. + * + * @param url the probe URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @param transport the transport whose pinning and custom request headers the probe reuses, + * {@code null} for a plain connection + * @return what the probe observed + */ + @Nonnull + @Override + public ProbeOutcome probe(@Nonnull String url, int timeoutMs, @Nullable Transport transport) { + ProbeOutcome outcome = new ProbeOutcome(); + long startNs = System.nanoTime(); + HttpURLConnection connection = null; + try { + connection = transport == null ? Transport.openBareGetConnection(url, timeoutMs) : transport.openProbeConnection(url, timeoutMs); + int status = connection.getResponseCode(); + if (status > 0) { + outcome.status = status; + drain(connection); + } else { + //not an HTTP answer + outcome.failure = errorTransport; + } + } catch (SocketTimeoutException e) { + outcome.status = 0; + outcome.failure = errorTimeout; + } catch (Exception e) { + outcome.status = 0; + outcome.failure = errorTransport; + } finally { + if (connection != null) { + connection.disconnect(); + } + outcome.ms = (System.nanoTime() - startNs) / 1_000_000L; + } + return outcome; + } + + /** + * Reads the body, up to {@link #MAX_DRAIN_BYTES}, and discards it. + * + * @param connection the answered connection + */ + private static void drain(@Nonnull HttpURLConnection connection) { + try (InputStream stream = bodyStream(connection)) { + if (stream == null) { + return; + } + byte[] buffer = new byte[4096]; + int total = 0; + int read; + while (total < MAX_DRAIN_BYTES && (read = stream.read(buffer)) != -1) { + total += read; + } + } catch (IOException ignored) { + //the status is already read, a failed body read changes nothing + } + } + + /** + * The body of the answer, which is the error stream for a failing status. + * + * @param connection the answered connection + * @return the body, {@code null} when there is none + */ + @Nullable + private static InputStream bodyStream(@Nonnull HttpURLConnection connection) { + try { + return connection.getInputStream(); + } catch (IOException e) { + return connection.getErrorStream(); + } + } + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java index c60d04d9..104e66ab 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import org.json.JSONArray; @@ -49,8 +50,16 @@ public class ModuleContent extends ModuleBase { private boolean fetching = false; private int waitForDelay = 0; private int generation = 0; - - private int zoneTimerInterval = ConfigContent.DEFAULT_ZONE_TIMER_INTERVAL; + /** + * Whether the active zone was entered because the {@code ecz} setting asked for it, so turning + * that setting off leaves it. Guarded by {@link #contentLock}. + */ + private boolean zoneEnteredBySettings = false; + /** + * The {@code ecz} value this module last acted on, so a settings change acts only when that + * value changed. Guarded by {@link #contentLock}. + */ + private boolean contentZoneEnabledApplied = false; /** When the SDK came up, which is what {@link #START_DELAY_MS} is measured from. */ private long startedAt = 0; @@ -64,10 +73,102 @@ public void init(InternalConfig config) { super.init(config); L.v("[ModuleContent] Initializing"); - zoneTimerInterval = config.content.zoneTimerInterval; globalContentCallback = config.content.globalContentCallback; contentInterface = new Content(); startedAt = TimeUtils.uniqueTimestampMs(); + synchronized (contentLock) { + contentZoneEnabledApplied = config.getConfigurationProvider().getContentZoneEnabled(); + } + } + + /** + * Enters the content zone when the {@code ecz} setting asks for it. While no display is + * registered the zone is entered once one is. + * + * @param config configuration of the running SDK + */ + @Override + protected void initFinished(InternalConfig config) { + super.initFinished(config); + if (config.getConfigurationProvider().getContentZoneEnabled()) { + enterContentZoneForSettings("initFinished"); + } + } + + /** + * Applies the content settings a server response changed. A changed {@code czi} restarts the + * timer of an active zone. A changed {@code ecz} enters the zone when it turned on, and when it + * turned off leaves the zone only if this setting entered it: a zone entered through + * {@link Content#enterContentZone()} is left to the application. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + ConfigurationProvider configProvider = config.getConfigurationProvider(); + boolean contentZoneEnabled = configProvider.getContentZoneEnabled(); + boolean contentZoneSettingChanged; + synchronized (contentLock) { + contentZoneSettingChanged = contentZoneEnabled != contentZoneEnabledApplied; + contentZoneEnabledApplied = contentZoneEnabled; + } + + restartZoneTimerIfIntervalChanged(configProvider.getContentZoneTimerInterval()); + + if (!contentZoneSettingChanged) { + return; + } + + if (contentZoneEnabled) { + enterContentZoneForSettings("onSdkConfigurationChanged"); + } else if (exitContentZoneInternal(true, true)) { + L.i("[ModuleContent] onSdkConfigurationChanged, the SDK behavior settings turned the content zone off, left the zone they had entered"); + } else { + L.d("[ModuleContent] onSdkConfigurationChanged, the SDK behavior settings turned the content zone off, no zone they entered is active"); + } + } + + /** + * Enters the content zone because the {@code ecz} setting asks for it, once a display is registered. + * + * @param caller the calling method, for the log + */ + private void enterContentZoneForSettings(@Nonnull String caller) { + boolean hasDisplay; + synchronized (contentLock) { + hasDisplay = display != null; + } + + if (!hasDisplay) { + L.d("[ModuleContent] " + caller + ", the SDK behavior settings ask for the content zone, it is entered once a content display is registered"); + return; + } + + L.d("[ModuleContent] " + caller + ", entering the content zone as the SDK behavior settings ask"); + enterContentZoneInternal(true); + } + + /** + * Restarts the timer of an active zone when the fetch interval changed, keeping the zone as it + * is. The next fetch comes as soon as the start delay allows. + * + * @param intervalSeconds the fetch interval now in effect + */ + private void restartZoneTimerIfIntervalChanged(int intervalSeconds) { + CountlyTimer timerToStop; + synchronized (contentLock) { + if (!zoneActive || contentTimer == null || contentTimer.getTimerDelaySeconds() == intervalSeconds) { + return; + } + + L.d("[ModuleContent] restartZoneTimerIfIntervalChanged, the fetch interval changed from [" + contentTimer.getTimerDelaySeconds() + "] to [" + intervalSeconds + "] seconds, restarting the zone timer"); + timerToStop = contentTimer; + contentTimer = new CountlyTimer(L); + contentTimer.startTimer(intervalSeconds, firstFetchDelay(), this::onZoneTimerTick); + } + + // Not awaited: the zone stays active, so a tick of the replaced timer that is still running is harmless. + timerToStop.stopTimer(false); } @Override @@ -105,12 +206,30 @@ protected void deviceIdChanged(String oldDeviceId, boolean withMerge) { void setContentDisplayInternal(ContentDisplay contentDisplay) { L.d("[ModuleContent] setContentDisplayInternal, display set:[" + (contentDisplay != null) + "]"); + boolean zoneInactive; synchronized (contentLock) { display = contentDisplay; + zoneInactive = !zoneActive; + } + + if (contentDisplay != null && zoneInactive && internalConfig.getConfigurationProvider().getContentZoneEnabled()) { + enterContentZoneForSettings("setContentDisplayInternal"); } } void enterContentZoneInternal() { + enterContentZoneInternal(false); + } + + /** + * Enters the content zone, fetching every {@code czi} seconds. Entering a zone that is already + * active does nothing, except that a call made for the application keeps a zone the settings + * entered active until the application leaves it. + * + * @param bySettings {@code true} when the {@code ecz} setting asks for the zone, {@code false} + * when the application does + */ + private void enterContentZoneInternal(boolean bySettings) { if (display == null) { L.w("[ModuleContent] enterContentZoneInternal, no content display is registered, ignoring the call"); return; @@ -121,9 +240,15 @@ void enterContentZoneInternal() { return; } + int zoneTimerInterval = internalConfig.getConfigurationProvider().getContentZoneTimerInterval(); synchronized (contentLock) { if (zoneActive) { - L.d("[ModuleContent] enterContentZoneInternal, already in a content zone, ignoring the call"); + if (!bySettings && zoneEnteredBySettings) { + zoneEnteredBySettings = false; + L.d("[ModuleContent] enterContentZoneInternal, already in the content zone the SDK behavior settings entered, it now stays until it is left through exitContentZone"); + } else { + L.d("[ModuleContent] enterContentZoneInternal, already in a content zone, ignoring the call"); + } return; } @@ -132,6 +257,7 @@ void enterContentZoneInternal() { contentShown = false; fetching = false; waitForDelay = 0; + zoneEnteredBySettings = bySettings; // Any fetch left in flight from a previous zone belongs to an older generation and is // discarded when it completes. generation++; @@ -172,13 +298,31 @@ void exitContentZoneInternal() { * tick, because a task cannot wait for itself to finish */ private void exitContentZoneInternal(boolean awaitTimerTermination) { + exitContentZoneInternal(awaitTimerTermination, false); + } + + /** + * Leaves the content zone. + * + * @param awaitTimerTermination must be {@code false} when called from the zone timer's own + * tick, because a task cannot wait for itself to finish + * @param onlyIfEnteredBySettings {@code true} to leave only an active zone the {@code ecz} + * setting entered + * @return whether the zone was left + */ + private boolean exitContentZoneInternal(boolean awaitTimerTermination, boolean onlyIfEnteredBySettings) { CountlyTimer timerToStop; synchronized (contentLock) { + if (onlyIfEnteredBySettings && !(zoneActive && zoneEnteredBySettings)) { + return false; + } + zoneActive = false; shouldFetch = false; contentShown = false; fetching = false; waitForDelay = 0; + zoneEnteredBySettings = false; generation++; timerToStop = contentTimer; @@ -191,14 +335,27 @@ private void exitContentZoneInternal(boolean awaitTimerTermination) { } L.i("[ModuleContent] exitContentZoneInternal, left the content zone"); + return true; } + /** + * Flushes the event queue and enters the content zone again, keeping who entered it. Called by + * the application and when the server accepted a journey trigger; ignored while the + * {@code rcz} setting forbids refreshing and while a content block is on screen. + */ void refreshContentZoneInternal() { + if (!internalConfig.getConfigurationProvider().getRefreshContentZoneEnabled()) { + L.d("[ModuleContent] refreshContentZoneInternal, refreshing the content zone is disabled by the SDK behavior settings, ignoring the call"); + return; + } + + boolean enteredBySettings; synchronized (contentLock) { if (contentShown) { L.d("[ModuleContent] refreshContentZoneInternal, a content block is on screen, ignoring the call"); return; } + enteredBySettings = zoneActive && zoneEnteredBySettings; } // Push whatever is queued out first, so the trigger the developer just recorded has a @@ -206,7 +363,7 @@ void refreshContentZoneInternal() { flushEventQueue(); exitContentZoneInternal(); - enterContentZoneInternal(); + enterContentZoneInternal(enteredBySettings); } void previewContentInternal(String contentId) { @@ -468,7 +625,8 @@ public class Content { /** * Register the display that draws content blocks. Required before entering a content zone. - * Pass {@code null} to unregister. + * Pass {@code null} to unregister. While the SDK behavior settings ask for the content zone, + * registering a display enters it. * * @param contentDisplay the display to draw content with * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes @@ -481,6 +639,8 @@ public void setContentDisplay(@Nullable ContentDisplay contentDisplay) { /** * Start asking the server for content to show. Ignored while already in a content zone. + * Once this is called, the zone stays even when the SDK behavior settings turn the content + * zone off. * * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ @@ -506,7 +666,8 @@ public void exitContentZone() { /** * Re-enter the content zone right away, after flushing the event queue. Use it when a - * trigger condition just changed. Ignored while a content block is on screen. + * trigger condition just changed. Ignored while a content block is on screen and while the + * SDK behavior settings do not allow refreshing the content zone. * * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java index 2dbfb15a..2c1365cc 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java @@ -3,6 +3,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import ly.count.sdk.java.Crash; import ly.count.sdk.java.CrashProcessor; @@ -17,7 +19,13 @@ public class ModuleCrashes extends ModuleBase { private boolean crashed = false; protected InternalConfig config; + private final Object crashHandlerLock = new Object(); + //guarded by crashHandlerLock private Thread.UncaughtExceptionHandler previousHandler = null; + //guarded by crashHandlerLock + private Thread.UncaughtExceptionHandler installedHandler = null; + //written under crashHandlerLock + private volatile boolean stopped = false; protected CrashProcessor crashProcessor = null; protected List logs = new ArrayList<>(); @@ -41,9 +49,7 @@ public void init(InternalConfig config) { @Override public void stop(InternalConfig config, boolean clear) { try { - if (previousHandler != null) { - Thread.setDefaultUncaughtExceptionHandler(previousHandler); - } + restoreUncaughtExceptionHandler(); if (clear) { config.sdk.sdkStorage.storablePurge(config, CrashImpl.getStoragePrefix()); } @@ -56,32 +62,127 @@ public void stop(InternalConfig config, boolean clear) { @Override public void initFinished(final InternalConfig config) { - previousHandler = Thread.getDefaultUncaughtExceptionHandler(); + installUncaughtExceptionHandlerIfEnabled("initFinished"); + started = System.nanoTime(); + } + + /** + * Installs the uncaught exception handler when a server response enabled automatic crash + * reporting and it is not installed yet. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + installUncaughtExceptionHandlerIfEnabled("onSdkConfigurationChanged"); + } - if (internalConfig.sdk.hasConsentForFeature(CoreFeature.CrashReporting) && config.isUnhandledCrashReportingEnabled()) { - registerUncaughtExceptionHandler(); + /** + * Installs the uncaught exception handler once, while crash reporting has consent and the SDK + * behavior settings enable both crash reporting and automatic crash reporting. An installed + * handler stays until the module stops and checks both settings again for every crash. + * + * @param caller the calling function, for the log + */ + private void installUncaughtExceptionHandlerIfEnabled(@Nonnull String caller) { + if (stopped) { + return; } - started = System.nanoTime(); + if (!isAutomaticCrashReportingEnabled()) { + L.d("[ModuleCrash] " + caller + ", automatic crash reporting is disabled, the uncaught exception handler is not installed"); + return; + } + + if (!internalConfig.sdk.hasConsentForFeature(CoreFeature.CrashReporting)) { + L.d("[ModuleCrash] " + caller + ", crash reporting has no consent, the uncaught exception handler is not installed"); + return; + } + + boolean installed = false; + synchronized (crashHandlerLock) { + if (installedHandler == null && !stopped) { + previousHandler = Thread.getDefaultUncaughtExceptionHandler(); + installedHandler = createUncaughtExceptionHandler(previousHandler); + Thread.setDefaultUncaughtExceptionHandler(installedHandler); + installed = true; + } + } + + if (installed) { + L.d("[ModuleCrash] " + caller + ", installed the uncaught exception handler"); + } + } + + /** + * Whether the SDK behavior settings enable both crash reporting and automatic crash reporting. + * + * @return {@code true} when unhandled crashes may be recorded + */ + private boolean isAutomaticCrashReportingEnabled() { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + return configProvider.getCrashReportingEnabled() && configProvider.getAutomaticCrashReportingEnabled(); } - private void registerUncaughtExceptionHandler() { - final Thread.UncaughtExceptionHandler handler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> { + /** + * Builds the handler that records an unhandled crash, while the module runs and the SDK behavior + * settings enable automatic crash reporting, and then always hands the crash to the handler it replaced. + * + * @param previous the handler to hand every crash to, {@code null} for none + * @return the handler + */ + @Nonnull + private Thread.UncaughtExceptionHandler createUncaughtExceptionHandler(@Nullable final Thread.UncaughtExceptionHandler previous) { + return (thread, throwable) -> { // needed since following UncaughtExceptionHandler can keep reference to this one crashed = true; - if (isActive()) { - recordExceptionInternal(throwable, false, null, null); + try { + if (isActive() && !stopped && isAutomaticCrashReportingEnabled()) { + recordExceptionInternal(throwable, false, null, null); + } + } catch (Throwable recordFailure) { + L.e("[ModuleCrash] uncaughtException, failed to record the crash, [" + recordFailure + "]"); + } finally { + if (previous != null) { + previous.uncaughtException(thread, throwable); + } } + }; + } - if (handler != null) { - handler.uncaughtException(thread, throwable); + /** + * Stops the installed handler from recording and puts back the handler it replaced, if it is + * still the default one. A handler installed over it keeps it in the chain, where it only hands + * crashes on. + */ + private void restoreUncaughtExceptionHandler() { + String outcome = null; + synchronized (crashHandlerLock) { + stopped = true; + if (installedHandler != null) { + if (Thread.getDefaultUncaughtExceptionHandler() == installedHandler) { + Thread.setDefaultUncaughtExceptionHandler(previousHandler); + outcome = "restored the uncaught exception handler it replaced"; + } else { + outcome = "another uncaught exception handler was installed over this one, which stays in the chain and only hands crashes on"; + } + installedHandler = null; + previousHandler = null; } - }); + } + + if (outcome != null) { + L.d("[ModuleCrash] stop, " + outcome); + } } protected void recordExceptionInternal(Throwable t, boolean handled, Map segments, String legacyCrashName) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getCrashReportingEnabled()) { + L.d("[ModuleCrash] recordExceptionInternal, crash reporting disabled by SDK behavior settings; ignoring"); + return; + } if (config.isBackendModeEnabled()) { L.w("[ModuleCrash] recordExceptionInternal, Skipping crash, backend mode is enabled!"); return; @@ -134,6 +235,9 @@ public CrashImpl onCrash(InternalConfig config, CrashImpl crash) { L.e("[ModuleCrash] Error when calling CrashProcessor#process(Crash)" + t); } } + + //after the processor, so it matches against the whole crash and what it adds is limited too + crash.applyInternalLimits(config.getConfigurationProvider(), "[ModuleCrash] onCrash"); if (!Storage.push(config, crash)) { L.e("[ModuleCrash] Couldn't persist a crash, so dumping it here: " + crash.getJSON()); } else { @@ -148,11 +252,16 @@ protected void addBreadcrumbInternal(String record) { return; } - if (logs.size() >= config.getMaxBreadcrumbCount()) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + String breadcrumb = UtilsInternalLimits.truncateValue(record, configProvider.getMaxValueSize(), L, "[ModuleCrash] addBreadcrumbInternal"); + + //a loop, as the limit can shrink while breadcrumbs are kept + int maxBreadcrumbCount = configProvider.getMaxBreadcrumbCount(); + while (!logs.isEmpty() && logs.size() >= maxBreadcrumbCount) { logs.remove(0); } - logs.add(record); + logs.add(breadcrumb); } public class Crashes { diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java index 1e48a875..e45c182e 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java @@ -1,18 +1,30 @@ package ly.count.sdk.java.internal; +import java.util.Collection; +import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import ly.count.sdk.java.Session; import ly.count.sdk.java.View; public class ModuleEvents extends ModuleBase { + static final String INTERNAL_EVENT_KEY_PREFIX = "[CLY]_"; + protected EventQueue eventQueue = null; final Map timedEvents = new ConcurrentHashMap<>(); protected Events eventsInterface = null; ViewIdProvider viewIdProvider = null; IdGenerator idGenerator = null; String previousEventId = null; + /** + * Storage IDs of the event requests that carry a journey trigger and are waiting for their first + * response, which decides whether the content zone is refreshed. + */ + final Set journeyTriggerRequestIds = ConcurrentHashMap.newKeySet(); @Override public void init(InternalConfig config) { @@ -41,6 +53,52 @@ public Boolean onRequest(Request request) { return true; } + /** + * Refreshes the content zone once the server accepted a request that carries a journey trigger. + * The first attempt to send such a request settles it: when it fails, neither it nor its retry + * refreshes anything. A request dropped from the queue unsent settles it the same way. + * + * @param request the request that was sent + * @param response the response body, {@code null} when none could be read + * @param responseCode the response code, {@link Transport#NO_RESPONSE_CODE} when no response arrived + */ + @Override + public void onRequestCompleted(Request request, String response, int responseCode) { + if (request == null || !journeyTriggerRequestIds.remove(request.storageId())) { + return; + } + + if (!Transport.isSuccessfulResponse(responseCode, response)) { + L.d("[ModuleEvents] onRequestCompleted, the request [" + request.storageId() + "] carrying a journey trigger failed with code [" + responseCode + "], the content zone is not refreshed"); + return; + } + + SDKCore core = SDKCore.instance; + ModuleContent content = core == null ? null : core.module(ModuleContent.class); + if (content == null) { + L.d("[ModuleEvents] onRequestCompleted, the server accepted the request [" + request.storageId() + "] carrying a journey trigger, there is no content zone to refresh"); + return; + } + + L.d("[ModuleEvents] onRequestCompleted, the server accepted the request [" + request.storageId() + "] carrying a journey trigger, refreshing the content zone"); + content.refreshContentZoneInternal(); + } + + /** + * Settles the journey triggers carried by requests that were removed from the queue unread, + * which leaves their owner unknown and untold. No response will ever arrive for them, and none + * of them refreshes the content zone. + * + * @param requestIds storage IDs of the removed requests + */ + void onRequestsDroppedUnread(@Nonnull Collection requestIds) { + for (Long requestId : requestIds) { + if (journeyTriggerRequestIds.remove(requestId)) { + L.d("[ModuleEvents] onRequestsDroppedUnread, the request [" + requestId + "] carrying a journey trigger was dropped from the queue without being sent, the content zone is not refreshed"); + } + } + } + @Override public void deviceIdChanged(String oldDeviceId, boolean withMerge) { super.deviceIdChanged(oldDeviceId, withMerge); @@ -69,13 +127,30 @@ public void deviceIdChanged(String oldDeviceId, boolean withMerge) { @Override public void stop(InternalConfig config, final boolean clear) { super.stop(config, clear); + journeyTriggerRequestIds.clear(); if (clear) { eventQueue.clear(); timedEvents.clear(); } } - private synchronized void addEventsToRequestQ(String deviceId) { + /** + * Moves every queued event into one request. + * + * @param deviceId the device ID the events belong to, {@code null} for the current one + */ + private void addEventsToRequestQ(@Nullable String deviceId) { + addEventsToRequestQ(deviceId, false); + } + + /** + * Moves every queued event into one request. + * + * @param deviceId the device ID the events belong to, {@code null} for the current one + * @param journeyTrigger whether the events include a journey trigger, whose request refreshes + * the content zone once the server accepts it + */ + private synchronized void addEventsToRequestQ(@Nullable String deviceId, boolean journeyTrigger) { L.d("[ModuleEvents] addEventsToRequestQ"); if (eventQueue.getEQ().isEmpty()) { @@ -91,10 +166,28 @@ private synchronized void addEventsToRequestQ(String deviceId) { request.own(ModuleEvents.class); eventQueue.clear(); - ModuleRequests.pushAsync(internalConfig, request); + if (!journeyTrigger) { + ModuleRequests.pushAsync(internalConfig, request); + return; + } + + final Long requestId = request.storageId(); + journeyTriggerRequestIds.add(requestId); + L.d("[ModuleEvents] addEventsToRequestQ, the request [" + requestId + "] carries a journey trigger, the content zone is refreshed once the server accepts it"); + ModuleRequests.pushAsync(internalConfig, request, false, stored -> { + if (!Boolean.TRUE.equals(stored)) { + journeyTriggerRequestIds.remove(requestId); + } + }); } protected void recordEventInternal(String key, int count, Double sum, Double dur, Map segmentation, String eventIdOverride) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled()) { + L.d("[ModuleEvents] recordEventInternal, tracking is disabled by the SDK behavior settings, event [" + key + "] will not be recorded"); + return; + } + if (count <= 0) { L.w("[ModuleEvents] recordEventInternal, Count can't be less than 1, ignoring this event."); return; @@ -105,10 +198,32 @@ protected void recordEventInternal(String key, int count, Double sum, Double dur return; } + boolean isCustomEvent = !key.startsWith(INTERNAL_EVENT_KEY_PREFIX); + if (isCustomEvent) { + if (!configProvider.getCustomEventTrackingEnabled()) { + L.d("[ModuleEvents] recordEventInternal, custom event tracking is disabled by the SDK behavior settings, event [" + key + "] will not be recorded"); + return; + } + + if (!UtilsListingFilters.applyEventFilter(key, configProvider)) { + L.w("[ModuleEvents] recordEventInternal, event [" + key + "] is filtered out by the event filter of the SDK behavior settings, it will not be recorded"); + return; + } + + segmentation = filteredCustomEventSegmentation(key, segmentation, configProvider); + } + + boolean journeyTrigger = isJourneyTrigger(key, isCustomEvent, segmentation, configProvider); + L.d("[ModuleEvents] recordEventInternal, Recording event with key: [" + key + "] and provided event ID of:[" + eventIdOverride + "] and segmentation with:[" + (segmentation == null ? "null" : segmentation.size()) + "] keys"); Utils.removeInvalidDataFromSegments(segmentation, L); + if (isCustomEvent) { + key = UtilsInternalLimits.truncateKey(key, configProvider.getMaxKeyLength(), L, "[ModuleEvents] recordEventInternal"); + segmentation = UtilsInternalLimits.applySegmentationLimits(segmentation, configProvider, L, "[ModuleEvents] recordEventInternal"); + } + if (internalConfig.isAutoSendUserProperties() && internalConfig.sdk.userProfile() != null) { internalConfig.sdk.module(ModuleUserProfile.class).saveInternal(); } @@ -134,19 +249,82 @@ protected void recordEventInternal(String key, int count, Double sum, Double dur this.previousEventId = eventId; } - addEventToQueue(new EventImpl(key, count, sum, dur, segmentation, L, eventId, pvid, cvid, previousEventIdToSend)); + addEventToQueue(new EventImpl(key, count, sum, dur, segmentation, L, eventId, pvid, cvid, previousEventIdToSend), journeyTrigger); } - private void addEventToQueue(EventImpl event) { + /** + * Applies the segmentation filter and then the segmentation filter of this event to the + * segmentation of a custom event. The filters work on a copy, as the map belongs to the caller. + * + * @param key the key of the custom event + * @param segmentation the segmentation the caller passed, {@code null} for none + * @param configProvider the settings in effect + * @return the filtered copy, {@code null} when there was no segmentation + */ + @Nullable + private Map filteredCustomEventSegmentation(@Nonnull String key, @Nullable Map segmentation, @Nonnull ConfigurationProvider configProvider) { + if (segmentation == null) { + return null; + } + + Map filtered = new HashMap<>(segmentation); + UtilsListingFilters.applySegmentationFilter(filtered, configProvider, L); + UtilsListingFilters.applyEventSegmentationFilter(key, filtered, configProvider, L); + return filtered; + } + + /** + * Whether an event is a journey trigger of the SDK behavior settings: a custom event whose key + * is in {@code jte}, or a view event whose name is in {@code jtv}. + * + * @param key the key of the event, as it was passed + * @param isCustomEvent whether the key is outside the {@code [CLY]_} prefix + * @param segmentation the segmentation of the event, {@code null} for none + * @param configProvider the settings in effect + * @return {@code true} for a journey trigger + */ + private static boolean isJourneyTrigger(@Nonnull String key, boolean isCustomEvent, @Nullable Map segmentation, @Nonnull ConfigurationProvider configProvider) { + if (isCustomEvent) { + return configProvider.getJourneyTriggerEvents().contains(key); + } + + if (!ModuleViews.KEY_VIEW_EVENT.equals(key) || segmentation == null) { + return false; + } + + Object viewName = segmentation.get(ModuleViews.KEY_NAME); + return viewName != null && configProvider.getJourneyTriggerViews().contains(viewName); + } + + /** + * Adds an event to the event queue and sends the queue when it is full or holds a journey trigger. + * + * @param event the event + * @param journeyTrigger whether the event is a journey trigger + */ + private void addEventToQueue(@Nonnull EventImpl event, boolean journeyTrigger) { L.d("[ModuleEvents] addEventToQueue"); eventQueue.addEvent(event); - checkEventQueueToSend(false); + if (journeyTrigger) { + L.d("[ModuleEvents] addEventToQueue, event [" + event.key + "] is a journey trigger, sending the event queue now"); + } + checkEventQueueToSend(journeyTrigger, journeyTrigger); } void checkEventQueueToSend(boolean forceSend) { + checkEventQueueToSend(forceSend, false); + } + + /** + * Sends the queued events when asked to or when the queue reached its threshold. + * + * @param forceSend whether to send whatever is queued + * @param journeyTrigger whether the queue holds a journey trigger + */ + private void checkEventQueueToSend(boolean forceSend, boolean journeyTrigger) { L.d("[ModuleEvents] queue size:[" + eventQueue.eqSize() + "] || forceSend: " + forceSend); - if (forceSend || eventQueue.eqSize() >= internalConfig.getEventsBufferSize()) { - addEventsToRequestQ(null); + if (forceSend || eventQueue.eqSize() >= internalConfig.getConfigurationProvider().getEventQueueSizeThreshold()) { + addEventsToRequestQ(null, journeyTrigger); } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java index d088a526..8b9747a6 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java @@ -263,7 +263,8 @@ private void reportFeedbackWidgetManuallyInternal(CountlyFeedbackWidget widgetIn } else { //widget was filled out //merge given segmentation - segm.putAll(widgetResult); + int maxValueSize = internalConfig.getConfigurationProvider().getMaxValueSize(); + segm.putAll(UtilsInternalLimits.truncateStringValues(widgetResult, maxValueSize, L, "[ModuleFeedback] reportFeedbackWidgetManuallyInternal")); } Countly.instance().events().recordEvent(widgetInfo.type.eventKey, segm); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java index c81b2adb..5dc0d858 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java @@ -39,12 +39,28 @@ void disableLocationInternal() { void sendLocation() { L.d("[ModuleLocation] Calling 'sendLocation'"); SessionImpl session = internalConfig.sdk.getSession(); - if (session == null || session.getBegan() != null) { + if (session == null || session.getBegan() != null || !isLocationSentWithSessionBegin()) { ModuleRequests.pushAsync(internalConfig, new Request(prepareLocationParams()), true, null); } // else case, values are added to the session begin request } + /** + * Whether a session that has not begun yet sends the location with its begin request, which the + * SDK behavior settings allow only while both session tracking and location tracking are enabled. + * + * @return {@code true} when the location can wait for the begin request + */ + private boolean isLocationSentWithSessionBegin() { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + return configProvider.getSessionTrackingEnabled() && configProvider.getLocationTrackingEnabled(); + } + void setLocationInternal(@Nullable String countryCode, @Nullable String cityName, @Nullable String gpsCoordinates, @Nullable String ipAddress) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getLocationTrackingEnabled()) { + L.d("[ModuleLocation] setLocationInternal, location tracking disabled by SDK behavior settings; ignoring"); + return; + } L.d("[ModuleLocation] setLocationInternal, Setting location parameters, cc[" + countryCode + "] cy[" + city + "] gps[" + gpsCoordinates + "] ip[" + ipAddress + "]"); if (countryCode != null ^ city != null) { @@ -96,8 +112,10 @@ public void stop(InternalConfig internalConfig, boolean clear) { @Override public void initFinished(@Nonnull InternalConfig config) { if (config.isLocationDisabled()) { - //disable location if needed - disableLocationInternal(); + //disable location if needed, unless a settings response during init already did and sent the erase request + if (!locationDisabled) { + disableLocationInternal(); + } } else { //if we are not disabling location, check for other set values String[] locParams = config.getLocationParams(); // country, city, location, ip @@ -107,6 +125,22 @@ public void initFinished(@Nonnull InternalConfig config) { } } + /** + * Disables location when a server response turned location tracking off, which erases the + * location stored on the server as {@link Location#disableLocation()} does. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + synchronized (Countly.instance()) { + if (!locationDisabled && !config.getConfigurationProvider().getLocationTrackingEnabled()) { + L.d("[ModuleLocation] onSdkConfigurationChanged, location tracking was disabled by the SDK behavior settings, disabling location"); + disableLocationInternal(); + } + } + } + protected void saveLocationToParamsLegacy(Params params) { if (countryLegacy != null) { params.add("country_code", countryLegacy); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java index 5ea633c1..af9f6184 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java @@ -1,7 +1,12 @@ package ly.count.sdk.java.internal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import javax.annotation.Nonnull; /** * Centralized place for all requests construction & handling. @@ -99,6 +104,11 @@ public static Future location(InternalConfig config, double latitude, d return null; } + if (!config.getConfigurationProvider().getLocationTrackingEnabled()) { + config.getLogger().d("[ModuleRequests] location, location tracking is disabled by the SDK behavior settings, the location is not sent"); + return null; + } + Request request = sessionRequest(config, null, null, null); request.params.add("location", latitude + "," + longitude); return pushAsync(config, request); @@ -195,21 +205,36 @@ public static Future pushAsync(InternalConfig config, Request request) } /** - * Common store-request logic: store & send a ping to the service. + * Common store-request logic: store & send a ping to the service. While the SDK behavior + * settings set the request queue size, storing a request drops the oldest ones the queue holds + * over that size. * * @param config InternalConfig to run in * @param request Request to store * @param noControl do not check empty validity of the request - * @param callback Callback (nullable) to call when storing is done, called in {@link Storage} {@link Thread} + * @param callback Callback (nullable) to call when storing is done, called in {@link Storage} {@link Thread}, + * or on the calling thread with {@code false} when the request is not stored * @return {@link Future} which resolves to {@code} true if stored successfully, false otherwise */ public static Future pushAsync(final InternalConfig config, final Request request, final boolean noControl, final Tasks.Callback callback) { config.getLogger().d("New request " + request.storageId() + ": " + request); + if (!config.getConfigurationProvider().getTrackingEnabled()) { + config.getLogger().d("[ModuleRequests] pushAsync, tracking disabled by SDK behavior settings; dropping request"); + if (callback != null) { + try { + callback.call(false); + } catch (Exception e) { + config.getLogger().e("[ModuleRequests] Exception in a callback " + e); + } + } + return null; + } + if (!noControl && request.isEmpty()) { if (callback != null) { try { - callback.call(null); + callback.call(false); } catch (Exception e) { config.getLogger().e("[ModuleRequests] Exception in a callback " + e); } @@ -221,10 +246,99 @@ public static Future pushAsync(final InternalConfig config, final Reque addRequiredParametersToParams(config, request.params); return Storage.pushAsync(config, request, param -> { + if (Boolean.TRUE.equals(param)) { + dropOldestRequestsOverQueueLimit(config); + } SDKCore.instance.onRequest(config, request); if (callback != null) { callback.call(param); } }); } + + /** + * Stores a request as it is, without the checks of {@link #pushAsync(InternalConfig, Request, boolean, Tasks.Callback)}, + * then drops the oldest requests the queue holds over the request queue size of the SDK behavior + * settings. Waits until both are done. + * + * @param config configuration of the running SDK + * @param request the request, with every parameter it is sent with + * @return whether the request was stored + */ + static boolean pushWithinQueueLimit(@Nonnull final InternalConfig config, @Nonnull final Request request) { + try { + return Boolean.TRUE.equals(Storage.pushAsync(config, request, stored -> { + if (Boolean.TRUE.equals(stored)) { + dropOldestRequestsOverQueueLimit(config); + } + }).get()); + } catch (InterruptedException | ExecutionException e) { + config.getLogger().e("[ModuleRequests] pushWithinQueueLimit, failed to store request " + request.storageId() + ", [" + e + "]"); + return false; + } + } + + /** + * Drops the oldest stored requests while the queue holds more than the request queue size of the + * SDK behavior settings. Without a size from the settings the queue has no limit. Must run on the + * storage thread, as it works on the stored files directly. The files are removed unread, so + * their owners are not told; the journey triggers they carried are settled instead. + * + * @param config configuration of the running SDK + * @return how many requests were dropped + */ + static int dropOldestRequestsOverQueueLimit(@Nonnull InternalConfig config) { + ConfigurationProvider configProvider = config.getConfigurationProvider(); + if (!configProvider.isRequestQueueMaxSizeFromBehaviorSettings()) { + return 0; + } + + List droppedIds = new ArrayList<>(); + try { + int maxSize = configProvider.getRequestQueueMaxSize(); + List requestIds = config.sdk.sdkStorage.storableList(config, Request.getStoragePrefix(), 0); + int overflow = requestIds.size() - maxSize; + if (overflow <= 0) { + return 0; + } + + Collections.sort(requestIds); + for (int i = 0; i < overflow; i++) { + Long requestId = requestIds.get(i); + if (Boolean.TRUE.equals(config.sdk.sdkStorage.storableRemove(config, new Request(requestId)))) { + droppedIds.add(requestId); + } + } + + config.getLogger().w("[ModuleRequests] dropOldestRequestsOverQueueLimit, the request queue went over its size of [" + maxSize + "] set by the SDK behavior settings, dropped the [" + droppedIds.size() + "] oldest requests"); + } catch (Exception e) { + config.getLogger().e("[ModuleRequests] dropOldestRequestsOverQueueLimit, failed to keep the request queue within its size, [" + e + "]"); + } + + settleJourneyTriggersOf(config, droppedIds); + return droppedIds.size(); + } + + /** + * Tells the events module which requests were dropped unread, so a journey trigger one of them + * carried stops waiting for a response. Only a set of IDs is touched, which is safe on the + * storage thread. + * + * @param config configuration of the running SDK + * @param droppedIds storage IDs of the dropped requests + */ + private static void settleJourneyTriggersOf(@Nonnull InternalConfig config, @Nonnull List droppedIds) { + if (droppedIds.isEmpty() || config.sdk == null) { + return; + } + + try { + ModuleEvents events = config.sdk.module(ModuleEvents.class); + if (events != null) { + events.onRequestsDroppedUnread(droppedIds); + } + } catch (Exception e) { + config.getLogger().e("[ModuleRequests] settleJourneyTriggersOf, failed to settle the journey triggers of the dropped requests, [" + e + "]"); + } + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java index 58977e5c..e4bfdcee 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java @@ -39,6 +39,11 @@ public void initFinished(final InternalConfig config) { @Override protected void onTimer() { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getSessionTrackingEnabled()) { + L.d("[ModuleSessions] onTimer, session tracking disabled by SDK behavior settings; skipping update"); + return; + } if (!internalConfig.isBackendModeEnabled() && isActive() && getSession() != null) { L.i("[ModuleSessions] onTimer, updating session"); getSession().update(); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java index 41aeb5da..bcbe4366 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java @@ -1,10 +1,18 @@ package ly.count.sdk.java.internal; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import ly.count.sdk.java.PredefinedUserPropertyKeys; import ly.count.sdk.java.User; @@ -15,6 +23,14 @@ public class ModuleUserProfile extends ModuleBase { static final String CUSTOM_KEY = "custom"; boolean isSynced = true; static final String PICTURE_BYTES = "[CLY]_picture_bytes"; + /** + * The keys of {@link PredefinedUserPropertyKeys}, which neither the user property filter nor the + * user property cache limit of the SDK behavior settings apply to. + */ + static final Set PREDEFINED_KEYS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + PredefinedUserPropertyKeys.NAME, PredefinedUserPropertyKeys.USERNAME, PredefinedUserPropertyKeys.EMAIL, + PredefinedUserPropertyKeys.ORGANIZATION, PredefinedUserPropertyKeys.PHONE, PredefinedUserPropertyKeys.PICTURE, + PredefinedUserPropertyKeys.PICTURE_PATH, PredefinedUserPropertyKeys.GENDER, PredefinedUserPropertyKeys.BIRTH_YEAR))); UserProfile userProfileInterface; private final Map sets; private final List ops; @@ -68,7 +84,7 @@ enum Op { } ModuleUserProfile() { - sets = new HashMap<>(); // keys should be nullable + sets = new LinkedHashMap<>(); // keys should be nullable, insertion order decides what the cache limit drops ops = new ArrayList<>(); } @@ -91,12 +107,23 @@ private Object optString(String key, Object value) { /** * Transforming changes in "sets" into a json contained in "changes" + *

+ * The SDK internal limits of the SDK behavior settings in effect apply here: string values of + * predefined properties, the picture excepted, are cut to the value size limit, and custom + * properties get the key length, value size and segmentation entry limits, the entry limit + * counting sets only, never modifications. * * @param changes JSONObject to store changes * @param params Params to store changes * @throws JSONException if something goes wrong */ void perform(JSONObject changes, Params params) throws JSONException { + ConfigurationProvider limits = internalConfig.getConfigurationProvider(); + int maxValueSize = limits.getMaxValueSize(); + Map customSets = supportedCustomSets(); + Map limitedCustomSets = UtilsInternalLimits.applySegmentationLimits(customSets, limits, L, "[ModuleUserProfile] perform"); + boolean customSetsLimited = !customSets.equals(limitedCustomSets); + for (String key : sets.keySet()) { Object value = sets.get(key); switch (key) { @@ -105,7 +132,7 @@ void perform(JSONObject changes, Params params) throws JSONException { case PredefinedUserPropertyKeys.EMAIL: case PredefinedUserPropertyKeys.ORGANIZATION: case PredefinedUserPropertyKeys.PHONE: - changes.put(key, optString(key, value)); + changes.put(key, optString(key, UtilsInternalLimits.truncateIfString(value, maxValueSize, L, "[ModuleUserProfile] perform"))); break; case PredefinedUserPropertyKeys.PICTURE: if (value == null) { @@ -137,6 +164,7 @@ void perform(JSONObject changes, Params params) throws JSONException { } break; case PredefinedUserPropertyKeys.GENDER: + value = UtilsInternalLimits.truncateIfString(value, maxValueSize, L, "[ModuleUserProfile] perform"); if (value == null || value instanceof User.Gender) { changes.put(PredefinedUserPropertyKeys.GENDER, value == null ? JSONObject.NULL : value.toString()); } else if (value instanceof String) { @@ -151,6 +179,7 @@ void perform(JSONObject changes, Params params) throws JSONException { } break; case PredefinedUserPropertyKeys.BIRTH_YEAR: + value = UtilsInternalLimits.truncateIfString(value, maxValueSize, L, "[ModuleUserProfile] perform"); if (value == null || value instanceof Integer) { changes.put(PredefinedUserPropertyKeys.BIRTH_YEAR, value == null ? JSONObject.NULL : value); } else if (value instanceof String) { @@ -164,25 +193,69 @@ void perform(JSONObject changes, Params params) throws JSONException { } break; default: - performCustomUpdate(key, value, changes); + if (!customSetsLimited || !customSets.containsKey(key)) { + performCustomUpdate(key, value, changes); + } break; } } - applyOps(changes); + if (customSetsLimited) { + for (Map.Entry entry : limitedCustomSets.entrySet()) { + performCustomUpdate(entry.getKey(), entry.getValue(), changes); + } + } + + applyOps(changes, limits); + } + + /** + * The pending sets of custom properties that {@link #performCustomUpdate} sends, the ones with a + * value of a supported type, in the order they were first set. + * + * @return the custom properties to send + */ + private Map supportedCustomSets() { + Map customSets = new LinkedHashMap<>(); + for (Map.Entry entry : sets.entrySet()) { + if (!isPredefinedKey(entry.getKey()) && isSupportedCustomValue(entry.getValue())) { + customSets.put(entry.getKey(), entry.getValue()); + } + } + return customSets; + } + + /** + * Whether a custom property can be sent with a value, which has to be of a supported type. + * + * @param value the value + * @return {@code true} for a supported type, {@code false} for any other type and for {@code null} + */ + private static boolean isSupportedCustomValue(@Nullable Object value) { + return value instanceof String || value instanceof Integer || value instanceof Float || value instanceof Double || value instanceof Boolean || value instanceof Object[]; } - private void applyOps(final JSONObject changes) throws JSONException { + /** + * Applies the pending modifications to the custom properties of the changes, with their keys cut + * to the key length limit and their string values cut to the value size limit. + * + * @param changes the changes to send + * @param limits the SDK behavior settings in effect + * @throws JSONException if a modification cannot be written + */ + private void applyOps(final JSONObject changes, @Nonnull ConfigurationProvider limits) throws JSONException { if (!ops.isEmpty() && !changes.has(CUSTOM_KEY)) { changes.put(CUSTOM_KEY, new JSONObject()); } for (OpParams opParam : ops) { - opParam.op.valueTransformer.apply(changes.getJSONObject(CUSTOM_KEY), opParam.key, opParam.value); + String key = UtilsInternalLimits.truncateKey(opParam.key, limits.getMaxKeyLength(), L, "[ModuleUserProfile] applyOps"); + Object value = UtilsInternalLimits.truncateIfString(opParam.value, limits.getMaxValueSize(), L, "[ModuleUserProfile] applyOps"); + opParam.op.valueTransformer.apply(changes.getJSONObject(CUSTOM_KEY), key, value); } } private void performCustomUpdate(final String key, final Object value, final JSONObject changes) throws JSONException { - if (value == null || value instanceof String || value instanceof Integer || value instanceof Float || value instanceof Double || value instanceof Boolean || value instanceof Object[]) { + if (value == null || isSupportedCustomValue(value)) { if (!changes.has(CUSTOM_KEY)) { changes.put(CUSTOM_KEY, new JSONObject()); } @@ -234,7 +307,14 @@ private void modifyCustomData(String key, Object value, Op mod) { L.w("[ModuleUserProfile] modifyCustomData, value is null, thus nothing to modify"); return; } + + if (!UtilsListingFilters.applyUserPropertyFilter(key, internalConfig.getConfigurationProvider())) { + L.w("[ModuleUserProfile] modifyCustomData, key [" + key + "] is filtered out by the user property filter of the SDK behavior settings, ignoring the modification"); + return; + } + ops.add(new OpParams(key, value, mod)); + applyCacheLimitToModifications(); isSynced = false; } @@ -250,10 +330,101 @@ protected void setPropertiesInternal(@Nonnull Map data) { return; } - sets.putAll(data); + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + Map accepted = new LinkedHashMap<>(); + for (Map.Entry entry : data.entrySet()) { + String key = entry.getKey(); + if (!isPredefinedKey(key) && !UtilsListingFilters.applyUserPropertyFilter(key, configProvider)) { + L.w("[ModuleUserProfile] setPropertiesInternal, key [" + key + "] is filtered out by the user property filter of the SDK behavior settings, it will not be set"); + continue; + } + accepted.put(key, entry.getValue()); + } + + if (accepted.isEmpty()) { + return; + } + + sets.putAll(accepted); + applyCacheLimitToCustomProperties(); isSynced = false; } + /** + * Whether a user property key is one of {@link PredefinedUserPropertyKeys}. + * + * @param key the key + * @return {@code true} for a predefined key + */ + private static boolean isPredefinedKey(@Nullable String key) { + return PREDEFINED_KEYS.contains(key); + } + + /** + * Keeps the pending custom properties within the user property cache limit of the SDK behavior + * settings by dropping the oldest ones. Predefined properties are not counted. + */ + private void applyCacheLimitToCustomProperties() { + int cacheLimit = internalConfig.getConfigurationProvider().getUserPropertyCacheLimit(); + if (sets.size() <= cacheLimit) { + return; + } + + int customCount = 0; + for (String key : sets.keySet()) { + if (!isPredefinedKey(key)) { + customCount++; + } + } + + int overflow = customCount - cacheLimit; + if (overflow <= 0) { + return; + } + + List dropped = new ArrayList<>(overflow); + Iterator keys = sets.keySet().iterator(); + while (keys.hasNext() && dropped.size() < overflow) { + String key = keys.next(); + if (!isPredefinedKey(key)) { + keys.remove(); + dropped.add(key); + } + } + + L.w("[ModuleUserProfile] applyCacheLimitToCustomProperties, [" + customCount + "] custom user properties are pending, over the cache limit of [" + cacheLimit + "] set by the SDK behavior settings, dropped the oldest ones: " + dropped); + } + + /** + * Keeps the keys with pending modifications within the user property cache limit of the SDK + * behavior settings by dropping every modification of the oldest keys. + */ + private void applyCacheLimitToModifications() { + int cacheLimit = internalConfig.getConfigurationProvider().getUserPropertyCacheLimit(); + if (ops.size() <= cacheLimit) { + return; + } + + Set modifiedKeys = new LinkedHashSet<>(); + for (OpParams op : ops) { + modifiedKeys.add(op.key); + } + + int overflow = modifiedKeys.size() - cacheLimit; + if (overflow <= 0) { + return; + } + + Set dropped = new LinkedHashSet<>(); + Iterator keys = modifiedKeys.iterator(); + while (dropped.size() < overflow) { + dropped.add(keys.next()); + } + ops.removeIf(op -> dropped.contains(op.key)); + + L.w("[ModuleUserProfile] applyCacheLimitToModifications, [" + modifiedKeys.size() + "] custom user properties have pending modifications, over the cache limit of [" + cacheLimit + "] set by the SDK behavior settings, dropped the modifications of the oldest ones: " + dropped); + } + protected void saveInternal() { Params generatedParams = prepareRequestParamsForUserProfile(); if (internalConfig.sdk.location() != null) { diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java index dd176c3e..289956e1 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java @@ -106,6 +106,7 @@ public void updateGlobalViewSegmentationInternal(@Nonnull Map se } private Map createViewEventSegmentation(@Nonnull ViewData vd, boolean firstView, boolean visit, Map customViewSegmentation) { + ConfigurationProvider limits = internalConfig.getConfigurationProvider(); Map viewSegmentation = new ConcurrentHashMap<>(); viewSegmentation.putAll(globalViewSegmentation); viewSegmentation.putAll(vd.viewSegmentation); @@ -114,7 +115,9 @@ private Map createViewEventSegmentation(@Nonnull ViewData vd, bo viewSegmentation.putAll(customViewSegmentation); } - viewSegmentation.put(KEY_NAME, vd.viewName); + viewSegmentation = applyInternalLimits(viewSegmentation, limits); + + viewSegmentation.put(KEY_NAME, UtilsInternalLimits.truncateKey(vd.viewName, limits.getMaxKeyLength(), L, "[ModuleViews] createViewEventSegmentation")); if (visit) { viewSegmentation.put(KEY_VISIT, KEY_VISIT_VALUE); } @@ -125,6 +128,30 @@ private Map createViewEventSegmentation(@Nonnull ViewData vd, bo return viewSegmentation; } + /** + * Applies the key length, value size and segmentation entry limits of the SDK behavior settings + * to the segmentation of a view event, before the SDK adds its own keys, which are neither cut + * nor counted. A key cut to one of the reserved keys is removed, and entries of unsupported + * types are removed before the entry limit counts, so they take no place under it. + * + * @param viewSegmentation the merged segmentation, which this module owns and may change + * @param limits the settings in effect + * @return the segmentation within the limits + */ + private Map applyInternalLimits(@Nonnull Map viewSegmentation, @Nonnull ConfigurationProvider limits) { + Map truncated = UtilsInternalLimits.truncateSegmentationKeysAndValues(viewSegmentation, limits.getMaxKeyLength(), limits.getMaxValueSize(), L, "[ModuleViews] createViewEventSegmentation"); + for (String reservedKey : reservedSegmentationKeysViews) { + if (truncated.remove(reservedKey) != null) { + L.w("[ModuleViews] createViewEventSegmentation, a key truncated to the reserved key [" + reservedKey + "] is removed"); + } + } + + if (truncated.size() > limits.getMaxSegmentationValues()) { + Utils.removeInvalidDataFromSegments(truncated, L); + } + return UtilsInternalLimits.limitSegmentationEntries(truncated, limits.getMaxSegmentationValues(), L, "[ModuleViews] createViewEventSegmentation"); + } + private void autoCloseRequiredViews(boolean closeAllViews, Map customViewSegmentation) { L.d("[ModuleViews] autoCloseRequiredViews"); List viewsToRemove = new ArrayList<>(); @@ -155,6 +182,11 @@ private void autoCloseRequiredViews(boolean closeAllViews, Map c * @return Returns link to Countly for call chaining */ @Nullable String startViewInternal(@Nullable String viewName, @Nullable Map customViewSegmentation, boolean viewShouldBeAutomaticallyStopped) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getViewTrackingEnabled()) { + L.d("[ModuleViews] startViewInternal, view tracking disabled by SDK behavior settings; ignoring"); + return null; + } if (viewName == null || viewName.isEmpty()) { L.e("[ModuleViews] startViewInternal, Trying to record view with null or empty view name, ignoring request"); @@ -208,6 +240,10 @@ void stopViewWithNameInternal(@Nullable String viewName, @Nullable Map customViewSegmentation) { + if (!isViewTrackingEnabled("stopViewWithIDInternal")) { + return; + } + ViewData vd = validateViewID(viewID, "stopViewWithIDInternal"); if (vd == null) { return; @@ -252,6 +288,10 @@ private void recordViewEndEvent(ViewData vd, @Nullable Map filte } void pauseViewWithIDInternal(String viewID) { + if (!isViewTrackingEnabled("pauseViewWithIDInternal")) { + return; + } + ViewData vd = validateViewID(viewID, "pauseViewWithIDInternal"); if (vd == null) { return; @@ -270,6 +310,10 @@ void pauseViewWithIDInternal(String viewID) { } void resumeViewWithIDInternal(String viewID) { + if (!isViewTrackingEnabled("resumeViewWithIDInternal")) { + return; + } + ViewData vd = validateViewID(viewID, "resumeViewWithIDInternal"); if (vd == null) { return; @@ -287,10 +331,29 @@ void resumeViewWithIDInternal(String viewID) { void stopAllViewsInternal(Map viewSegmentation) { L.d("[ModuleViews] stopAllViewsInternal"); + if (!isViewTrackingEnabled("stopAllViewsInternal")) { + return; + } autoCloseRequiredViews(true, viewSegmentation); } + /** + * Whether the SDK behavior settings allow view tracking. While they do not, stopping, pausing + * and resuming views is ignored, so every view keeps its state until view tracking is allowed again. + * + * @param function the ignored call, for the log + * @return {@code true} when view tracking is enabled + */ + private boolean isViewTrackingEnabled(@Nonnull String function) { + if (internalConfig.getConfigurationProvider().getViewTrackingEnabled()) { + return true; + } + + L.d("[ModuleViews] " + function + ", view tracking is disabled by the SDK behavior settings, ignoring the call"); + return false; + } + private ViewData validateViewID(String viewID, String function) { if (viewID == null || viewID.isEmpty()) { L.e("[ModuleViews] validateViewID, " + function + ", Trying to process view with null or empty view ID, ignoring request"); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java index 3c8aa193..d2d8769e 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java @@ -21,7 +21,10 @@ public class SDKCore { protected Networking networking; protected Queue requestQueueMemory = null; protected final Object lockBRQStorage = new Object(); - private CountlyTimer countlyTimer; + private final Object lockTimer = new Object(); + //guarded by lockTimer for writes + volatile CountlyTimer countlyTimer; + private ModuleConfiguration moduleConfiguration; protected Log L = null; protected static ModuleBase testDummyModule = null;//set during testing when trying to check the SDK's lifecycle @@ -106,7 +109,14 @@ public void stop(final boolean clear) { networking.stop(config); } - countlyTimer.stopTimer(); + CountlyTimer timerToStop; + synchronized (lockTimer) { + timerToStop = countlyTimer; + countlyTimer = null; + } + if (timerToStop != null) { + timerToStop.stopTimer(); + } L.i("[SDKCore] Stopping Countly SDK" + (clear ? " and clearing all data" : "")); @@ -124,6 +134,7 @@ public void stop(final boolean clear) { sdkStorage.stop(config, clear);//from original super class user = null; + moduleConfiguration = null; config = null; instance = null; } @@ -256,6 +267,10 @@ protected void buildModules(InternalConfig config, int features) throws IllegalA // standard required internal features modules.put(-3, new ModuleDeviceIdCore()); modules.put(-2, new ModuleRequests()); + if (moduleConfiguration != null) { + modules.put(-1, moduleConfiguration); + } + modules.put(0, new ModuleConnectionTest()); modules.put(CoreFeature.Sessions.getIndex(), new ModuleSessions()); modules.put(CoreFeature.UserProfiles.getIndex(), new ModuleUserProfile()); @@ -405,10 +420,6 @@ public void init(final InternalConfig givenConfig) { //setup module mapping prepareMappings(); - //create internal timer - countlyTimer = new CountlyTimer(L); - countlyTimer.startTimer(config.getSendUpdateEachSeconds(), this::onTimer); - //setup and perform migrations MigrationHelper migrationHelper = new MigrationHelper(L); migrationHelper.setupMigrations(config.storageProvider); @@ -418,6 +429,17 @@ public void init(final InternalConfig givenConfig) { setDeviceIdFromStorageIfExist(config); + //resolve the SDK behavior settings before anything reads them + moduleConfiguration = new ModuleConfiguration(); + moduleConfiguration.init(config); + applyLoggingSetting(config, moduleConfiguration); + applyConsentRequirementAtInit(config, moduleConfiguration.getConsentRequired()); + + //create internal timer + synchronized (lockTimer) { + startTimer(moduleConfiguration.getSessionUpdateInterval()); + } + requestQueueMemory = new ArrayDeque<>(config.getRequestQueueMaxSize()); if (config.viewIdGenerator == null) { @@ -449,7 +471,9 @@ public void init(final InternalConfig givenConfig) { modules.forEach((feature, module) -> { try { - module.init(config); + if (module != moduleConfiguration) { + module.init(config); + } module.setActive(true); } catch (IllegalArgumentException | IllegalStateException e) { L.e("[SDKCore] Error during module initialization" + e); @@ -530,6 +554,105 @@ private void initFinished(final InternalConfig config) { checkNetworking(config); } + /** + * Makes the consent requirement the SDK behavior settings resolved the one every module reads + * through {@link InternalConfig#requiresConsent()}. It is applied here, before the modules are + * built, and only here: a value that arrives later takes effect on the next init. + * + * @param config configuration of the SDK being initialized + * @param consentRequired the resolved consent requirement + */ + private void applyConsentRequirementAtInit(@Nonnull InternalConfig config, boolean consentRequired) { + if (consentRequired != config.requiresConsent()) { + L.i("[SDKCore] applyConsentRequirementAtInit, SDK behavior settings set the consent requirement to [" + consentRequired + "]"); + config.setRequiresConsent(consentRequired); + } + } + + /** + * Starts the global timer, replacing the running one. The caller holds {@link #lockTimer}. + * + * @param intervalSeconds seconds between two ticks + */ + private void startTimer(int intervalSeconds) { + countlyTimer = new CountlyTimer(L); + countlyTimer.startTimer(intervalSeconds, this::onTimer); + } + + /** + * Restarts the global timer when its interval differs from the given one. Never waits for the + * replaced timer, as a settings change can be applied from a timer task. + * + * @param intervalSeconds the session update interval now in effect + */ + private void restartTimerIfIntervalChanged(int intervalSeconds) { + synchronized (lockTimer) { + CountlyTimer runningTimer = countlyTimer; + if (runningTimer == null || runningTimer.getTimerDelaySeconds() == intervalSeconds) { + return; + } + + L.d("[SDKCore] restartTimerIfIntervalChanged, session update interval changed from [" + runningTimer.getTimerDelaySeconds() + "] to [" + intervalSeconds + "] seconds, restarting the timer"); + runningTimer.stopTimer(false); + startTimer(intervalSeconds); + } + } + + /** + * Applies the {@code log} switch of the SDK behavior settings to what the SDK logger prints. + * + * @param config configuration whose logger prints + * @param provider the settings in effect + */ + private void applyLoggingSetting(@Nonnull InternalConfig config, @Nonnull ConfigurationProvider provider) { + Log logger = config.getLogger(); + if (logger == null) { + return; + } + + logger.setLoggingEnabled(provider.getLoggingEnabled()); + logger.d("[SDKCore] applyLoggingSetting, logging enabled:[" + provider.getLoggingEnabled() + "], printing from level:[" + logger.getPrintLevel() + "]"); + } + + /** + * Applies SDK behavior settings that a server response changed while the SDK runs: sets what the + * logger prints, restarts the global timer when the session update interval changed, lets the + * request queue drain when networking is allowed, then notifies every module, each on its own. + * + * @param changedConfig configuration whose settings changed + */ + void onSdkConfigurationChanged(@Nonnull InternalConfig changedConfig) { + if (config == null || config != changedConfig) { + L.d("[SDKCore] onSdkConfigurationChanged, the SDK was stopped before the change arrived, ignoring it"); + return; + } + + L.i("[SDKCore] onSdkConfigurationChanged"); + ConfigurationProvider provider = changedConfig.getConfigurationProvider(); + applyLoggingSetting(changedConfig, provider); + restartTimerIfIntervalChanged(provider.getSessionUpdateInterval()); + + if (provider.getNetworkingEnabled()) { + checkNetworking(changedConfig); + } + + List modulesToNotify; + try { + modulesToNotify = new ArrayList<>(modules.values()); + } catch (RuntimeException e) { + L.e("[SDKCore] onSdkConfigurationChanged, failed to list the modules, [" + e + "]"); + return; + } + + for (ModuleBase module : modulesToNotify) { + try { + module.onSdkConfigurationChanged(changedConfig); + } catch (Exception e) { + L.e("[SDKCore] onSdkConfigurationChanged, " + module.getClass().getSimpleName() + " failed to apply the change, [" + e + "]"); + } + } + } + public UserImpl user() { return user; } @@ -642,8 +765,8 @@ public Boolean isRequestReady(Request request) { if (cls == null) { return true; } else { + // The owner stays on the request: Transport.send reads it to hand over the response, then removes it before sending ModuleBase module = module(cls); - request.params.remove(Request.MODULE); if (module == null) { return true; } else { @@ -655,7 +778,7 @@ public Boolean isRequestReady(Request request) { /** * After a network request has been finished * propagate that response to the module - * that owns the request + * that owns the request. A module that fails to handle it never fails the send. * * @param request the request that was sent, used to identify the request */ @@ -664,7 +787,11 @@ public void onRequestCompleted(Request request, String response, int responseCod ModuleBase module = module(requestOwner); if (module != null) { - module.onRequestCompleted(request, response, responseCode); + try { + module.onRequestCompleted(request, response, responseCode); + } catch (Exception e) { + L.e("[SDKCore] onRequestCompleted, " + module.getClass().getSimpleName() + " failed to handle the response of request [" + (request == null ? null : request.storageId()) + "], [" + e + "]"); + } } } } @@ -723,7 +850,7 @@ private boolean processCrash(InternalConfig config, Long id) { ModuleRequests.addRequiredParametersToParams(config, request.params); ModuleRequests.addRequiredTimeParametersToParams(request.params); - if (Storage.push(config, request)) { + if (ModuleRequests.pushWithinQueueLimit(config, request)) { L.i("[SDKCore] Added request " + request.storageId() + " instead of crash " + crash.storageId()); checkNetworking(config); Boolean success = Storage.remove(config, crash); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java index 4575608e..f4f2bf7e 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java @@ -33,6 +33,7 @@ public class SDKStorage implements StorageProvider { protected static final String key_device_id_type = "did_t"; protected static final String key_remote_config = "rc"; protected static final String key_migration_version = "dv"; + protected static final String key_server_config = "sc"; private JsonFileStorage jsonFileStorage; @@ -386,6 +387,20 @@ public void setMigrationVersion(Integer migrationVersion) { jsonFileStorage.addAndSave(key_migration_version, migrationVersion); } + @Override + public void setServerConfig(String config) { + if (config == null) { + jsonFileStorage.deleteAndSave(key_server_config); + } else { + jsonFileStorage.addAndSave(key_server_config, config); + } + } + + @Override + public String getServerConfig() { + return jsonFileStorage.getString(key_server_config); + } + @Override public boolean isCountlyStorageEmpty() { String[] files = getCountlyFileList(config); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java index 295bbe7b..ef12ca33 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java @@ -107,6 +107,8 @@ Future begin(Long now) { if (SDKCore.instance == null) { L.e("[SessionImpl] Countly is not initialized"); return null; + } else if (!isSessionTrackingEnabled("begin")) { + return null; } else if (began != null) { L.e("[SessionImpl] Session already began"); return null; @@ -125,8 +127,9 @@ Future begin(Long now) { if (pushOnChange) { Storage.pushAsync(config, this); } - if (hasConsent(CoreFeature.Location) && config.sdk.module(ModuleLocation.class) != null) { - params.add(config.sdk.module(ModuleLocation.class).prepareLocationParams()); + ModuleLocation locationModule = config.sdk.module(ModuleLocation.class); + if (hasConsent(CoreFeature.Location) && locationModule != null && config.getConfigurationProvider().getLocationTrackingEnabled()) { + params.add(locationModule.prepareLocationParams()); } Future ret = ModuleRequests.sessionBegin(config, this); @@ -151,6 +154,8 @@ Future update(Long now) { if (SDKCore.instance == null) { L.e("[SessionImpl] Countly is not initialized"); return null; + } else if (!isSessionTrackingEnabled("update")) { + return null; } else if (began == null) { L.e("[SessionImpl] Session is not began to update it"); return null; @@ -188,6 +193,8 @@ Future end(Long now, final Tasks.Callback callback, String did if (SDKCore.instance == null) { L.e("[SessionImpl] Countly is not initialized"); return null; + } else if (!isSessionTrackingEnabled("end")) { + return null; } else if (began == null) { L.e("[SessionImpl] Session is not began to end it"); return null; @@ -222,10 +229,22 @@ Future end(Long now, final Tasks.Callback callback, String did return ret; } + /** + * Ends a session left on disk by an earlier run, or removes it when there is nothing to end. + * While session tracking is disabled by the SDK behavior settings, no session request may be + * sent, so an unended session is removed without an end request: keeping it would end it on a + * later run with a stale duration. + * + * @param config configuration of the SDK being initialized + * @return whether the session was ended or removed, {@code null} when it is left as it is + */ Boolean recover(InternalConfig config) { Log L = config.getLogger(); if ((System.currentTimeMillis() - id) < 0) { return null; + } else if (!config.getConfigurationProvider().getSessionTrackingEnabled()) { + L.d("[SessionImpl] recover, session tracking is disabled by the SDK behavior settings, removing session " + id + " without ending it"); + return Storage.remove(config, this); } else { Future future; if (began == null) { @@ -260,6 +279,22 @@ public boolean isActive() { return began != null && ended == null; } + /** + * Whether the SDK behavior settings allow session tracking. While they do not, beginning, + * updating and ending a session are ignored, and the session keeps its state. + * + * @param function the ignored call, for the log + * @return {@code true} when session tracking is enabled + */ + private boolean isSessionTrackingEnabled(String function) { + if (config.getConfigurationProvider().getSessionTrackingEnabled()) { + return true; + } + + L.d("[SessionImpl] " + function + ", session tracking is disabled by the SDK behavior settings, ignoring the call"); + return false; + } + /** * Calculate time since last {@link #update()} or since {@link #begin()} if no {@link #update()} yet made, * set {@link #updated} to now. diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java index 1983dd7a..da2d7bd4 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java @@ -69,4 +69,18 @@ public interface StorageProvider { * @return true if empty, false otherwise */ boolean isCountlyStorageEmpty(); + + /** + * Persist the most-recently-validated SDK behavior settings JSON. + * + * @param config serialized configuration object, or {@code null} to clear + */ + void setServerConfig(String config); + + /** + * Read the persisted SDK behavior settings JSON. + * + * @return serialized configuration, or {@code null} if none persisted yet + */ + String getServerConfig(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java index e745e30b..02b1f771 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java @@ -30,6 +30,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.Future; +import java.util.function.LongConsumer; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; @@ -52,6 +55,12 @@ //class Network extends ModuleBase { - may be public class Transport implements X509TrustManager { + /** + * The response code handed to the module that owns a request when an attempt to send it ended + * before any response arrived. + */ + static final int NO_RESPONSE_CODE = -1; + private Log L = null; private static final String PARAMETER_TAMPERING_DIGEST = "SHA-256"; private static final String CHECKSUM = "checksum256"; @@ -144,21 +153,12 @@ HttpURLConnection connection(final Request request) throws IOException { connection.setConnectTimeout(1000 * config.getNetworkConnectionTimeout()); connection.setReadTimeout(1000 * config.getNetworkReadTimeout()); - if (connection instanceof HttpsURLConnection && sslContext != null) { - HttpsURLConnection https = (HttpsURLConnection) connection; - https.setSSLSocketFactory(sslContext.getSocketFactory()); - } + usePinningSocketFactory(connection); if (config.getCustomNetworkRequestHeaders() != null) { //if there are custom header values, add them L.v("[Transport] connection, Adding [" + config.getCustomNetworkRequestHeaders() + "] custom header fields"); - for (Map.Entry entry : config.getCustomNetworkRequestHeaders().entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (key != null && value != null && !key.isEmpty()) { - connection.addRequestProperty(key, value); - } - } + addCustomRequestHeaders(connection); } if (!usingGET) { @@ -221,6 +221,80 @@ HttpURLConnection connection(final Request request) throws IOException { return connection; } + /** + * Opens the GET of one connection test probe: a bare request, built as + * {@link #openBareGetConnection(String, int)} builds it, that verifies the server against the + * configured pins and carries the custom request headers, as every queued request does. Nothing + * is sent before the response is read. + * + * @param url the probe URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @return the connection, not connected yet + * @throws IOException when the connection cannot be opened + */ + HttpURLConnection openProbeConnection(@Nonnull String url, int timeoutMs) throws IOException { + HttpURLConnection connection = openBareGetConnection(url, timeoutMs); + usePinningSocketFactory(connection); + addCustomRequestHeaders(connection); + return connection; + } + + /** + * Opens a GET that carries nothing but its URL, follows no redirect and uses no cache. The + * redirect setting is made on this connection only, never through the default of every + * connection. + * + * @param url the URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @return the connection, not connected yet + * @throws IOException when the connection cannot be opened + */ + static HttpURLConnection openBareGetConnection(@Nonnull String url, int timeoutMs) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("GET"); + connection.setInstanceFollowRedirects(false); + connection.setUseCaches(false); + connection.setConnectTimeout(timeoutMs); + connection.setReadTimeout(timeoutMs); + connection.setDoInput(true); + connection.setDoOutput(false); + return connection; + } + + /** + * Makes an HTTPS connection verify the server against the configured pins, when pins are + * configured. + * + * @param connection the connection, not connected yet + */ + private void usePinningSocketFactory(HttpURLConnection connection) { + if (connection instanceof HttpsURLConnection && sslContext != null) { + HttpsURLConnection https = (HttpsURLConnection) connection; + https.setSSLSocketFactory(sslContext.getSocketFactory()); + } + } + + /** + * Adds the custom request headers of the configuration, skipping any with a {@code null} or + * empty name or a {@code null} value. + * + * @param connection the connection, not connected yet + */ + private void addCustomRequestHeaders(HttpURLConnection connection) { + Map headers = config.getCustomNetworkRequestHeaders(); + if (headers == null) { + return; + } + + for (Map.Entry entry : headers.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key != null && value != null && !key.isEmpty()) { + connection.addRequestProperty(key, value); + } + } + } + void addMultipart(OutputStream output, PrintWriter writer, final String boundary, final String contentType, final String name, final String value, final byte[] file) throws IOException { writer.append("--").append(boundary).append(Utils.CRLF); if (file != null) { @@ -300,7 +374,26 @@ String response(HttpURLConnection connection) { } } + /** + * Sends a request once and hands its response to the module that owns it. + * + * @param request request to send + * @return the task sending the request, resolving to whether the server accepted it + */ public Tasks.Task send(final Request request) { + return send(request, null); + } + + /** + * Sends a request as {@link #send(Request)} does, and tells how long the server took to answer + * it: from opening the connection, which sends the request, to reading the whole response. + * + * @param request request to send + * @param responseTimeListener given the response time in milliseconds once a response was read, + * never called when none was; {@code null} for none + * @return the task sending the request, resolving to whether the server accepted it + */ + Tasks.Task send(final Request request, @Nullable final LongConsumer responseTimeListener) { return new Tasks.Task(request.storageId()) { @Override public Boolean call() { @@ -311,16 +404,21 @@ public Boolean send() { L.i("[network] [send] Sending request: " + request); HttpURLConnection connection = null; + Class requestOwner = null; try { - Class requestOwner = request.owner(); + requestOwner = request.owner(); request.params.remove(Request.MODULE); + long sendStartNs = System.nanoTime(); connection = connection(request); connection.connect(); int code = connection.getResponseCode(); String response = response(connection); + if (responseTimeListener != null) { + responseTimeListener.accept((System.nanoTime() - sendStartNs) / 1_000_000L); + } try { if (request.params.has(Params.PARAM_OLD_DEVICE_ID) || request.params.has("token_session")) { @@ -341,6 +439,7 @@ public Boolean send() { return processResponse(code, response, request.storageId()); } catch (IOException e) { L.w("[network] Error while sending request " + request + " " + e); + notifyOwnerOfAttemptWithoutResponse(request, requestOwner); return false; } catch (Exception e) { L.e("[network] Unexpected error while sending request " + request + " " + e); @@ -354,6 +453,57 @@ public Boolean send() { }; } + /** + * Tells the module that owns a request that an attempt to send it ended before a response + * arrived, with a {@code null} response and {@link #NO_RESPONSE_CODE}. + * + * @param request the request + * @param requestOwner the module that owns it, {@code null} for none + */ + private void notifyOwnerOfAttemptWithoutResponse(Request request, Class requestOwner) { + SDKCore core = SDKCore.instance; + if (core == null || requestOwner == null) { + return; + } + + try { + core.onRequestCompleted(request, null, NO_RESPONSE_CODE, requestOwner); + } catch (Exception e) { + L.e("[network] Failed to tell the owner of request [" + request.storageId() + "] that it got no response, [" + e + "]"); + } + } + + /** + * Whether a response means the server accepted the request: a 2xx code and a JSON object body + * that has a {@code result}, the rule {@link #processResponse(int, String, Long)} applies. + * + * @param code the response code + * @param response the response body, {@code null} when none arrived + * @return {@code true} when the request was accepted + */ + static boolean isSuccessfulResponse(int code, String response) { + if (response == null) { + return false; + } + + try { + return isSuccessfulResponse(code, new JSONObject(response)); + } catch (Exception e) { + return false; + } + } + + /** + * Whether a parsed response means the server accepted the request. + * + * @param code the response code + * @param response the parsed response body + * @return {@code true} for a 2xx code with a {@code result} + */ + private static boolean isSuccessfulResponse(int code, JSONObject response) { + return code >= 200 && code < 300 && response.has("result"); + } + Boolean processResponse(int code, String response, Long requestId) { L.i("[network] [processResponse] Code [" + code + "] response [" + response + "] for request[" + requestId + "]"); @@ -364,7 +514,7 @@ Boolean processResponse(int code, String response, Long requestId) { try { JSONObject jsonObject = new JSONObject(response); - if (code >= 200 && code < 300 && jsonObject.has("result")) { + if (isSuccessfulResponse(code, jsonObject)) { L.d("[network] Success"); return true; } else { diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsInternalLimits.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsInternalLimits.java new file mode 100644 index 00000000..bca84bc4 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsInternalLimits.java @@ -0,0 +1,307 @@ +package ly.count.sdk.java.internal; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Applies the SDK internal limits of the SDK behavior settings: the key length {@code lkl}, the value + * size {@code lvs}, the number of segmentation entries {@code lsv}, the stack trace lines per thread + * {@code ltlpt} and the stack trace line length {@code ltl}. {@link Integer#MAX_VALUE} is no limit, + * and whatever no limit cuts is returned as the same instance. + *

+ * A map is never changed in place. One that has to change is rebuilt in sorted key order, so the + * outcome never depends on the order of the map: when two keys are cut to the same key the one that + * sorts last keeps its value, and over the entry limit the entries whose keys sort first are kept. + * Strings are cut in UTF-16 characters and never inside a surrogate pair. Every cut is logged as a + * warning that names the caller. + */ +final class UtilsInternalLimits { + + /** + * The order every limit walks keys in, a {@code null} key first. + */ + private static final Comparator KEY_ORDER = Comparator.nullsFirst(Comparator.naturalOrder()); + + /** + * Not instantiated: the limits are static helpers. + */ + private UtilsInternalLimits() { + } + + /** + * Cuts a key to the key length limit, {@code lkl}: an event key, a view name, a segmentation key + * or a custom user property key. + * + * @param key the key, {@code null} for none + * @param maxKeyLength the limit + * @param L logger + * @param tag the caller, for the log + * @return the key, cut when it is longer than the limit + */ + @Nullable + static String truncateKey(@Nullable String key, int maxKeyLength, @Nonnull Log L, @Nonnull String tag) { + if (key == null || key.length() <= maxKeyLength) { + return key; + } + + String truncated = truncate(key, maxKeyLength); + L.w(tag + ", the key [" + key + "] is longer than the key length limit of [" + maxKeyLength + "] of the SDK behavior settings, it is truncated to [" + truncated + "]"); + return truncated; + } + + /** + * Cuts a string value to the value size limit, {@code lvs}. + * + * @param value the value, {@code null} for none + * @param maxValueSize the limit + * @param L logger + * @param tag the caller, for the log + * @return the value, cut when it is longer than the limit + */ + @Nullable + static String truncateValue(@Nullable String value, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (value == null || value.length() <= maxValueSize) { + return value; + } + + String truncated = truncate(value, maxValueSize); + L.w(tag + ", a value of [" + value.length() + "] characters is longer than the value size limit of [" + maxValueSize + "] of the SDK behavior settings, it is truncated to [" + truncated + "]"); + return truncated; + } + + /** + * Cuts a value to the value size limit, {@code lvs}, when it is a string. A value of any other + * type is returned as it is, strings inside arrays included. + * + * @param value the value, {@code null} for none + * @param maxValueSize the limit + * @param L logger + * @param tag the caller, for the log + * @return the value, cut when it is a string longer than the limit + */ + @Nullable + static Object truncateIfString(@Nullable Object value, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (value instanceof String) { + return truncateValue((String) value, maxValueSize, L, tag); + } + return value; + } + + /** + * Applies the key length, value size and segmentation entry limits, {@code lkl}, {@code lvs} and + * {@code lsv}, to a segmentation. Keys and values are cut first, so two keys that end up the same + * are counted once. + * + * @param segmentation the segmentation, {@code null} for none, never changed + * @param limits the settings in effect + * @param L logger + * @param tag the caller, for the log + * @return the segmentation within the limits, the same instance when it already was + */ + @Nullable + static Map applySegmentationLimits(@Nullable Map segmentation, @Nonnull ConfigurationProvider limits, @Nonnull Log L, @Nonnull String tag) { + Map truncated = truncateSegmentationKeysAndValues(segmentation, limits.getMaxKeyLength(), limits.getMaxValueSize(), L, tag); + return limitSegmentationEntries(truncated, limits.getMaxSegmentationValues(), L, tag); + } + + /** + * Cuts the keys of a segmentation to the key length limit, {@code lkl}, and its string values to + * the value size limit, {@code lvs}. When two keys are cut to the same key, the entry whose key + * sorts last keeps its value. + * + * @param segmentation the segmentation, {@code null} for none, never changed + * @param maxKeyLength the key length limit + * @param maxValueSize the value size limit + * @param L logger + * @param tag the caller, for the log + * @return a copy in sorted key order when a key or a value was cut, otherwise the same instance + */ + @Nullable + static Map truncateSegmentationKeysAndValues(@Nullable Map segmentation, int maxKeyLength, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (segmentation == null || !exceedsKeyOrValueLimit(segmentation, maxKeyLength, maxValueSize)) { + return segmentation; + } + + Map truncated = new LinkedHashMap<>(); + for (String key : sortedKeys(segmentation.keySet())) { + String truncatedKey = truncateKey(key, maxKeyLength, L, tag); + if (truncated.containsKey(truncatedKey)) { + L.w(tag + ", the key [" + key + "] is truncated to the key of another segmentation entry, its value replaces that of [" + truncatedKey + "]"); + } + truncated.put(truncatedKey, truncateIfString(segmentation.get(key), maxValueSize, L, tag)); + } + return truncated; + } + + /** + * Keeps a segmentation within the segmentation entry limit, {@code lsv}, by keeping the entries + * whose keys sort first. + * + * @param segmentation the segmentation, {@code null} for none, never changed + * @param maxSegmentationValues the limit + * @param L logger + * @param tag the caller, for the log + * @return a copy in sorted key order when entries were dropped, otherwise the same instance + */ + @Nullable + static Map limitSegmentationEntries(@Nullable Map segmentation, int maxSegmentationValues, @Nonnull Log L, @Nonnull String tag) { + if (segmentation == null || segmentation.size() <= maxSegmentationValues) { + return segmentation; + } + + List keys = sortedKeys(segmentation.keySet()); + Map kept = new LinkedHashMap<>(); + for (String key : keys.subList(0, maxSegmentationValues)) { + kept.put(key, segmentation.get(key)); + } + + L.w(tag + ", the segmentation has [" + keys.size() + "] entries, over the limit of [" + maxSegmentationValues + "] of the SDK behavior settings, dropped the entries of the keys that sort last: " + keys.subList(maxSegmentationValues, keys.size())); + return kept; + } + + /** + * Cuts the string values of a map to the value size limit, {@code lvs}, leaving its keys and its + * number of entries as they are. + * + * @param map the map, never changed + * @param maxValueSize the limit + * @param L logger + * @param tag the caller, for the log + * @return a copy in the order of the map when a value was cut, otherwise the same instance + */ + @Nonnull + static Map truncateStringValues(@Nonnull Map map, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (!exceedsKeyOrValueLimit(map, Integer.MAX_VALUE, maxValueSize)) { + return map; + } + + Map truncated = new LinkedHashMap<>(); + for (Map.Entry entry : map.entrySet()) { + truncated.put(entry.getKey(), truncateIfString(entry.getValue(), maxValueSize, L, tag)); + } + return truncated; + } + + /** + * Cuts every line of a stack trace to the stack trace line length limit, {@code ltl}. Line breaks, + * {@code \n} or {@code \r\n}, are kept as they are and do not count as characters of the line. + * + * @param stackTrace the stack trace + * @param maxLineLength the limit + * @param L logger + * @param tag the caller, for the log + * @return the stack trace, a new one when a line was cut + */ + @Nonnull + static String truncateStackTraceLines(@Nonnull String stackTrace, int maxLineLength, @Nonnull Log L, @Nonnull String tag) { + StringBuilder truncated = null; + int truncatedLines = 0; + int lineStart = 0; + while (lineStart < stackTrace.length()) { + int lineBreak = stackTrace.indexOf('\n', lineStart); + int nextLineStart = lineBreak < 0 ? stackTrace.length() : lineBreak + 1; + int lineEnd = lineBreak < 0 ? stackTrace.length() : lineBreak; + if (lineEnd > lineStart && stackTrace.charAt(lineEnd - 1) == '\r') { + lineEnd--; + } + + if (lineEnd - lineStart > maxLineLength) { + if (truncated == null) { + truncated = new StringBuilder(stackTrace.length()); + truncated.append(stackTrace, 0, lineStart); + } + truncated.append(truncate(stackTrace.substring(lineStart, lineEnd), maxLineLength)).append(stackTrace, lineEnd, nextLineStart); + truncatedLines++; + } else if (truncated != null) { + truncated.append(stackTrace, lineStart, nextLineStart); + } + lineStart = nextLineStart; + } + + if (truncated == null) { + return stackTrace; + } + + L.w(tag + ", [" + truncatedLines + "] stack trace lines are longer than the line length limit of [" + maxLineLength + "] of the SDK behavior settings, they are truncated"); + return truncated.toString(); + } + + /** + * How many lines of the stack trace of one thread the stack trace lines per thread limit, + * {@code ltlpt}, keeps: the top ones. + * + * @param lineCount the number of lines of the stack trace + * @param maxLinesPerThread the limit + * @param threadName the thread, for the log + * @param L logger + * @param tag the caller, for the log + * @return the number of top lines to keep + */ + static int stackTraceLinesToKeep(int lineCount, int maxLinesPerThread, @Nullable String threadName, @Nonnull Log L, @Nonnull String tag) { + if (lineCount <= maxLinesPerThread) { + return lineCount; + } + + L.w(tag + ", the stack trace of the thread [" + threadName + "] has [" + lineCount + "] lines, over the limit of [" + maxLinesPerThread + "] lines per thread of the SDK behavior settings, dropped the bottom ones"); + return maxLinesPerThread; + } + + /** + * Whether a key of a map is longer than a key length limit or one of its string values is longer + * than a value size limit. + * + * @param map the map + * @param maxKeyLength the key length limit + * @param maxValueSize the value size limit + * @return {@code true} when a key or a value has to be cut + */ + private static boolean exceedsKeyOrValueLimit(@Nonnull Map map, int maxKeyLength, int maxValueSize) { + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (key != null && key.length() > maxKeyLength) { + return true; + } + if (value instanceof String && ((String) value).length() > maxValueSize) { + return true; + } + } + return false; + } + + /** + * The keys of a map in the order every limit walks them. + * + * @param keys the keys + * @return the keys, sorted + */ + @Nonnull + private static List sortedKeys(@Nonnull Collection keys) { + List sorted = new ArrayList<>(keys); + sorted.sort(KEY_ORDER); + return sorted; + } + + /** + * Cuts a string that is longer than a limit. A cut never splits a surrogate pair: it falls before + * the pair, or after it when the pair starts the string, which is then one character over the limit. + * + * @param value the string, longer than the limit + * @param limit the limit, at least 1 + * @return the start of the string + */ + @Nonnull + private static String truncate(@Nonnull String value, int limit) { + int end = limit; + if (Character.isHighSurrogate(value.charAt(end - 1)) && Character.isLowSurrogate(value.charAt(end))) { + end = end == 1 ? 2 : end - 1; + } + return value.substring(0, end); + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsListingFilters.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsListingFilters.java new file mode 100644 index 00000000..2d62ab45 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsListingFilters.java @@ -0,0 +1,122 @@ +package ly.count.sdk.java.internal; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Applies the listing filters of the SDK behavior settings: the event filter, the user property + * filter, the segmentation filter and the segmentation filter of each event. A filter without names + * allows everything; otherwise a whitelist keeps only its names and a blacklist removes its names. + */ +final class UtilsListingFilters { + + /** + * Not instantiated: the filters are static helpers. + */ + private UtilsListingFilters() { + } + + /** + * Whether the event filter, {@code eb} or {@code ew}, lets a custom event be recorded. + * + * @param eventKey the key of the custom event + * @param configProvider the settings in effect + * @return {@code true} when the event may be recorded + */ + static boolean applyEventFilter(@Nonnull String eventKey, @Nonnull ConfigurationProvider configProvider) { + return applyListFilter(eventKey, configProvider.getEventFilterList()); + } + + /** + * Whether the user property filter, {@code upb} or {@code upw}, lets a custom user property be + * set or modified. + * + * @param propertyKey the key of the custom user property + * @param configProvider the settings in effect + * @return {@code true} when the property may be set or modified + */ + static boolean applyUserPropertyFilter(@Nullable String propertyKey, @Nonnull ConfigurationProvider configProvider) { + return applyListFilter(propertyKey, configProvider.getUserPropertyFilterList()); + } + + /** + * Removes from a segmentation the keys the segmentation filter, {@code sb} or {@code sw}, does not allow. + * + * @param segmentation the segmentation, changed in place + * @param configProvider the settings in effect + * @param L logger + */ + static void applySegmentationFilter(@Nonnull Map segmentation, @Nonnull ConfigurationProvider configProvider, @Nonnull Log L) { + if (segmentation.isEmpty()) { + return; + } + + ConfigurationProvider.FilterList> filter = configProvider.getSegmentationFilterList(); + applyMapFilter(segmentation, filter.getFilterList(), filter.isWhitelist(), L); + } + + /** + * Removes from the segmentation of an event the keys that the segmentation filter of that event, + * {@code esb} or {@code esw}, does not allow. An event without names in the filter keeps every key. + * + * @param eventKey the key of the event + * @param segmentation the segmentation, changed in place + * @param configProvider the settings in effect + * @param L logger + */ + static void applyEventSegmentationFilter(@Nonnull String eventKey, @Nonnull Map segmentation, @Nonnull ConfigurationProvider configProvider, @Nonnull Log L) { + ConfigurationProvider.FilterList>> filter = configProvider.getEventSegmentationFilterList(); + if (segmentation.isEmpty() || filter.getFilterList().isEmpty()) { + return; + } + + Set names = filter.getFilterList().get(eventKey); + if (names == null) { + return; + } + + applyMapFilter(segmentation, names, filter.isWhitelist(), L); + } + + /** + * Removes the keys of a map that a filter does not allow. + * + * @param map the map, changed in place + * @param names the names of the filter + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + * @param L logger + */ + private static void applyMapFilter(@Nonnull Map map, @Nonnull Set names, boolean isWhitelist, @Nonnull Log L) { + if (names.isEmpty()) { + return; + } + + Iterator> entries = map.entrySet().iterator(); + while (entries.hasNext()) { + String key = entries.next().getKey(); + if (isWhitelist != names.contains(key)) { + entries.remove(); + L.d("[UtilsListingFilters] applyMapFilter, removed the segmentation key [" + key + "], " + (isWhitelist ? "it is not in the whitelist" : "it is in the blacklist")); + } + } + } + + /** + * Whether a filter allows a name. + * + * @param name the name + * @param filter the filter + * @return {@code true} when the filter has no names or allows this one + */ + private static boolean applyListFilter(@Nullable String name, @Nonnull ConfigurationProvider.FilterList> filter) { + Set names = filter.getFilterList(); + if (names.isEmpty()) { + return true; + } + + return filter.isWhitelist() == names.contains(name); + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java index 8111d5dc..bf047fa1 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java @@ -4,6 +4,9 @@ import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.json.JSONObject; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -238,6 +241,113 @@ public void deleteAndSave() { validateStorageSize(0, 1); } + /** + * "getJsonObj" hands out a deep copy. + *

+ * Verifies that changing the returned object, and an object and an array nested in it, changes + * neither what the storage returns next nor what "save" writes; that a missing key and a value + * that is not an object return the default; that the changed copy is stored once it is handed back + * through "addAndSave"; and that saving on one thread while another changes its copy and stores it, + * as the remote config values are, never fails to serialize and leaves every key on disk. + */ + @Test + public void getJsonObj_returnsADeepCopy_soAChangeReachesTheStorageOnlyThroughAdd() throws InterruptedException { + Log logger = Mockito.mock(Log.class); + setupJsonFile("{\"rc\":{\"color\":{\"v\":\"red\",\"c\":1},\"list\":[{\"a\":1}]},\"plain\":\"text\"}"); + storage = new JsonFileStorage(jsonFile(), logger); + + JSONObject copy = storage.getJsonObj("rc", null); + copy.getJSONObject("color").put("c", 0); + copy.getJSONArray("list").getJSONObject(0).put("a", 2); + copy.put("size", "large"); + + JSONObject stored = storage.getJsonObj("rc", null); + Assert.assertNotSame(copy, stored); + Assert.assertEquals(1, stored.getJSONObject("color").getInt("c")); + Assert.assertEquals(1, stored.getJSONArray("list").getJSONObject(0).getInt("a")); + Assert.assertFalse(stored.has("size")); + storage.save(); + Assert.assertTrue(stored.similar(TestUtils.readJsonFile(jsonFile()).getJSONObject("rc"))); + + JSONObject fallback = new JSONObject(); + Assert.assertSame(fallback, storage.getJsonObj("missing", fallback)); + Assert.assertSame(fallback, storage.getJsonObj("plain", fallback)); + Assert.assertNull(storage.getJsonObj("missing", null)); + + storage.addAndSave("rc", copy); + Assert.assertTrue(copy.similar(storage.getJsonObj("rc", null))); + Assert.assertTrue(copy.similar(TestUtils.readJsonFile(jsonFile()).getJSONObject("rc"))); + + storage.add("device", "device_id"); + List failures = new CopyOnWriteArrayList<>(); + long deadline = System.currentTimeMillis() + 400; + Thread changer = new Thread(() -> { + try { + for (int round = 0; System.currentTimeMillis() < deadline; round++) { + JSONObject values = storage.getJsonObj("rc", new JSONObject()); + for (int i = 0; i < 50; i++) { + values.put("key_" + ((round * 50 + i) % 400), new JSONObject().put("v", i).put("c", 1)); + } + for (int i = 0; i < 20; i++) { + values.remove("key_" + ((round * 37 + i) % 400)); + } + storage.add("rc", values); + } + } catch (Throwable t) { + failures.add(t); + } + }); + changer.start(); + try { + while (System.currentTimeMillis() < deadline) { + storage.save(); + } + } catch (Throwable t) { + failures.add(t); + } + changer.join(); + + Assert.assertTrue("failures: " + failures, failures.isEmpty()); + Mockito.verify(logger, Mockito.never()).e(Mockito.startsWith("[JsonFileStorage] save, Failed to serialize")); + storage.save(); + JSONObject onDisk = TestUtils.readJsonFile(jsonFile()); + Assert.assertEquals("device_id", onDisk.getString("device")); + Assert.assertEquals("text", onDisk.getString("plain")); + Assert.assertTrue(onDisk.getJSONObject("rc").similar(storage.getJsonObj("rc", null))); + } + + /** + * "save" when the data cannot be serialized. + *

+ * Verifies that the file keeps what it had instead of being emptied, and that the failure is logged. + */ + @Test + public void save_leavesTheFileAsItIs_whenTheDataCannotBeSerialized() { + Log logger = Mockito.mock(Log.class); + setupJsonFile("{\"device\":\"device_id\"}"); + storage = new JsonFileStorage(jsonFile(), logger); + storage.add("broken", new JSONObject() { + /** + * Fails as a value that cannot be serialized does. + * + * @param writer the writer + * @param indentFactor the indentation of each level + * @param indent the indentation of this level + * @return never + */ + @Override + public java.io.Writer write(java.io.Writer writer, int indentFactor, int indent) { + throw new IllegalStateException("cannot be written"); + } + }); + + storage.save(); + + Assert.assertEquals("device_id", TestUtils.readJsonFile(jsonFile()).getString("device")); + Assert.assertFalse(TestUtils.readJsonFile(jsonFile()).has("broken")); + Mockito.verify(logger).e(Mockito.startsWith("[JsonFileStorage] save, Failed to serialize the data")); + } + private static File jsonFile() { return new File(TestUtils.getTestSDirectory(), JSON_FILE_NAME); } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationContentTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationContentTests.java new file mode 100644 index 00000000..1e1617b2 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationContentTests.java @@ -0,0 +1,767 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.function.BooleanSupplier; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The content settings and the journey triggers of the SDK behavior settings: {@code ecz}, + * {@code czi}, {@code rcz}, {@code jte} and {@code jtv}, observed through the public API. + *

+ * Settings requests and content fetches go through a {@link SettingsAndContentServer}, which answers + * on the calling thread. The zone timer's first tick is held an hour away, so the zone fetches only + * when a test ticks it by hand, and a tick that fetches proves the zone is active. The journey + * scenarios send the event requests through the real networking stack to a local HTTP server, since + * the content zone is refreshed only after the server answers the request carrying the trigger. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationContentTests { + + private static final String CONTENT_ENDPOINT = "/o/sdk/content?"; + private static final String NO_CONTENT_RESPONSE = "{\"jsonArray\":[{\"result\":\"No content block found!\"}]}"; + private static final String CONTENT_RESPONSE = "{\"html\":\"https://content.count.ly/block-1\",\"geo\":{" + + "\"p\":{\"x\":10,\"y\":20,\"w\":300,\"h\":400},\"l\":{\"x\":30,\"y\":40,\"w\":500,\"h\":600}}}"; + private static final long WAIT_BUDGET_MS = 30_000; + + private SettingsAndContentServer server; + private QueueServer queueServer; + private FakeDisplay display; + private final List logs = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module and with fresh fakes. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new SettingsAndContentServer(); + queueServer = null; + display = new FakeDisplay(); + logs.clear(); + } + + /** + * Stops the SDK, clearing its data, and the local HTTP server when one was started. + */ + @After + public void afterTest() { + Countly.instance().halt(); + if (queueServer != null) { + queueServer.http.stop(0); + } + } + + // region scenarios + + /** + * {@code ecz} enters the content zone once a display is registered, and turning it off leaves + * only a zone it entered. + *

+ * Verifies that the response of the fetch made at init, applied before the content module + * finished its init, enters nothing while no display is registered; that registering one enters + * the zone, which a refresh keeps as the settings' zone; that turning the setting off leaves that + * zone and turning it on enters again; that a zone the application entered, or claimed by entering + * while the settings' zone ran, survives the setting turning off; that a response which leaves the + * setting as it is changes nothing, so a zone the application left stays left; that registering a + * display again enters; and that the stored setting enters again after a restart. + */ + @Test + public void contentZoneEnabled_entersOnceADisplayIsRegistered_andTurningItOffLeavesOnlyTheZoneItEntered() { + server.settings.respondWith(new ServerConfigBuilder().contentZone(true)); + init(contentConfig()); + holdTheZoneTimer(); + + Assert.assertTrue(provider().getContentZoneEnabled()); + Assert.assertEquals(2, logsContaining("it is entered once a content display is registered")); + assertZoneIsInactive(0); + + Countly.instance().content().setContentDisplay(display); + assertZoneFetches(1); + Countly.instance().content().refreshContentZone(); + assertZoneFetches(2); + + push(new ServerConfigBuilder().contentZone(false)); + assertZoneIsInactive(2); + push(new ServerConfigBuilder().contentZone(true)); + assertZoneFetches(3); + + Countly.instance().content().enterContentZone(); + push(new ServerConfigBuilder().contentZone(false)); + assertZoneFetches(4); + push(new ServerConfigBuilder().eventQueueSize(5)); + assertZoneFetches(5); + Countly.instance().content().exitContentZone(); + assertZoneIsInactive(5); + + Countly.instance().content().enterContentZone(); + push(new ServerConfigBuilder().contentZone(true)); + push(new ServerConfigBuilder().contentZone(false)); + assertZoneFetches(6); + Countly.instance().content().exitContentZone(); + + push(new ServerConfigBuilder().contentZone(true)); + assertZoneFetches(7); + Countly.instance().content().exitContentZone(); + push(new ServerConfigBuilder().contentZone(true).eventQueueSize(6)); + assertZoneIsInactive(7); + + Countly.instance().content().setContentDisplay(display); + assertZoneFetches(8); + Countly.instance().stop(); + + server.settings.pending(); + init(contentConfig()); + holdTheZoneTimer(); + assertZoneIsInactive(8); + Countly.instance().content().setContentDisplay(display); + assertZoneFetches(9); + } + + /** + * {@code czi} is the fetch interval of a zone, read when the zone is entered. + *

+ * Verifies that without the setting the developer's interval is used, that the setting overrides + * it, that a response changing it restarts the timer of the active zone at the new interval while + * the zone keeps fetching, that a response repeating it keeps the running timer, and that a change + * while no zone is active starts nothing and applies when a zone is entered. + */ + @Test + public void contentZoneInterval_isReadWhenTheZoneIsEntered_andAChangeRestartsTheTimerOfTheActiveZone() { + server.settings.pending(); + init(contentConfig().content.setZoneTimerInterval(20)); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(display); + Countly.instance().content().enterContentZone(); + Assert.assertEquals(20, zoneTimer().getTimerDelaySeconds()); + Countly.instance().halt(); + + server.settings.respondWith(new ServerConfigBuilder().contentZoneInterval(60)); + init(contentConfig().content.setZoneTimerInterval(20)); + holdTheZoneTimer(); + Assert.assertEquals(60, provider().getContentZoneTimerInterval()); + Countly.instance().content().setContentDisplay(display); + Countly.instance().content().enterContentZone(); + CountlyTimer atEnter = zoneTimer(); + Assert.assertEquals(60, atEnter.getTimerDelaySeconds()); + assertZoneFetches(1); + + push(new ServerConfigBuilder().contentZoneInterval(90)); + CountlyTimer restarted = zoneTimer(); + Assert.assertNotSame(atEnter, restarted); + Assert.assertEquals(90, restarted.getTimerDelaySeconds()); + assertZoneFetches(2); + + push(new ServerConfigBuilder().contentZoneInterval(90).eventQueueSize(3)); + Assert.assertSame(restarted, zoneTimer()); + + Countly.instance().content().exitContentZone(); + push(new ServerConfigBuilder().contentZoneInterval(120)); + Assert.assertNull(zoneTimer()); + Countly.instance().content().enterContentZone(); + Assert.assertEquals(120, zoneTimer().getTimerDelaySeconds()); + assertZoneFetches(3); + } + + /** + * {@code jte} and {@code jtv} send the whole event queue as soon as a trigger is recorded. + *

+ * Verifies on disk, with networking off, that events that are not triggers wait for the queue + * threshold, an internal event listed in {@code jte} included, that a custom trigger sends them all + * with it, that each later trigger gets its own request and each such request waits for its + * response, that both the start and the end of a trigger view send the queue, and that a trigger + * filtered out by {@code eb} or a trigger view while view tracking is off sends nothing. + */ + @Test + public void journeyTriggers_sendTheWholeEventQueueRightAway_eachTriggerInItsOwnRequest() { + server.settings.respondWith(new ServerConfigBuilder() + .journeyTriggerEvents(names("purchase", "[CLY]_star_rating")) + .journeyTriggerViews(names("checkout")) + .eventQueueSize(100) + .networking(false)); + init(contentConfig().enableFeatures(Config.Feature.Views)); + + Countly.instance().events().recordEvent("regular"); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Countly.instance().views().startView("home"); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + TestUtils.validateEQSize(3); + + Countly.instance().events().recordEvent("purchase", TestUtils.map("item", "book")); + TestUtils.letTheClockCatchUp(); + TestUtils.validateEQSize(0); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + List sent = TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID); + Assert.assertEquals(4, sent.size()); + Assert.assertEquals("regular", sent.get(0).key); + Assert.assertEquals("[CLY]_star_rating", sent.get(1).key); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, sent.get(2).key); + Assert.assertEquals("purchase", sent.get(3).key); + Assert.assertEquals("book", sent.get(3).segmentation.get("item")); + + Countly.instance().events().recordEvent("purchase"); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Assert.assertEquals(1, TestUtils.readEventsFromRequest(1, TestUtils.DEVICE_ID).size()); + + Countly.instance().views().startView("checkout"); + TestUtils.letTheClockCatchUp(); + Countly.instance().views().stopViewWithName("checkout"); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(4, TestUtils.getCurrentRQ().length); + List checkoutStart = TestUtils.readEventsFromRequest(2, TestUtils.DEVICE_ID); + List checkoutEnd = TestUtils.readEventsFromRequest(3, TestUtils.DEVICE_ID); + Assert.assertEquals("checkout", checkoutStart.get(0).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertEquals("1", checkoutStart.get(0).segmentation.get(ModuleViews.KEY_VISIT)); + Assert.assertEquals("checkout", checkoutEnd.get(0).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertNull(checkoutEnd.get(0).segmentation.get(ModuleViews.KEY_VISIT)); + Assert.assertEquals(storedRequestIds(), SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds); + + Countly.instance().views().startView("profile"); + push(new ServerConfigBuilder().eventFilterList(names("purchase"), false)); + Countly.instance().events().recordEvent("purchase"); + push(new ServerConfigBuilder().viewTracking(false)); + Assert.assertNull(Countly.instance().views().startView("checkout")); + Assert.assertEquals(4, TestUtils.getCurrentRQ().length); + TestUtils.validateEQSize(1); + Assert.assertEquals(4, SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds.size()); + } + + /** + * A {@code jte} trigger refreshes the content zone once the server accepts the request carrying it. + *

+ * Verifies against a local server that the zone is entered only after the request carrying the + * trigger and the events queued before it was accepted; that while a content block is on screen + * the refresh is ignored; that a rejected request refreshes nothing, and neither does its retry + * once accepted; and that an attempt that got no response settles it the same way. + */ + @Test + public void journeyTriggerEvent_refreshesTheZoneOnlyWhenTheServerAcceptsItsRequest() throws Exception { + startQueueServer(); + server.settings.respondWith(new ServerConfigBuilder().journeyTriggerEvents(names("purchase"))); + init(networkedContentConfig()); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(display); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("regular"); + TestUtils.validateEQSize(1); + Countly.instance().events().recordEvent("purchase"); + awaitZoneFetch(1); + Assert.assertEquals(1, queueServer.eventRequests().size()); + Assert.assertEquals("[\"regular\",\"purchase\"]", eventKeys(queueServer.eventRequests().get(0)).toString()); + awaitEmptyQueue(); + + server.contentResponse = CONTENT_RESPONSE; + assertZoneFetches(2); + Assert.assertEquals(1, display.presented.size()); + Countly.instance().events().recordEvent("purchase"); + awaitLogs("a content block is on screen, ignoring the call", 1); + display.lastCallback.onClosed(new HashMap<>()); + Countly.instance().content().exitContentZone(); + server.contentResponse = NO_CONTENT_RESPONSE; + awaitEmptyQueue(); + assertZoneIsInactive(2); + + queueServer.mode = QueueServer.REJECT; + Countly.instance().events().recordEvent("purchase"); + awaitLogs("carrying a journey trigger failed with code [500]", 1); + Assert.assertEquals(1, queuedRequestCount()); + assertZoneIsInactive(2); + queueServer.mode = QueueServer.ACCEPT; + awaitEmptyQueue(); + assertZoneIsInactive(2); + + queueServer.mode = QueueServer.DROP; + Countly.instance().events().recordEvent("purchase"); + awaitLogs("carrying a journey trigger failed with code [" + Transport.NO_RESPONSE_CODE + "]", 1); + assertZoneIsInactive(2); + queueServer.mode = QueueServer.ACCEPT; + awaitEmptyQueue(); + assertZoneIsInactive(2); + Assert.assertEquals(2, logsContaining("carrying a journey trigger, refreshing the content zone")); + Assert.assertTrue(SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds.isEmpty()); + } + + /** + * A {@code jtv} trigger refreshes the zone like an event does, it never triggers while view + * tracking is off, and {@code rcz} off ignores every refresh. + *

+ * Verifies against a local server that with refreshing forbidden the application's refresh is + * ignored before it sends the queued events and the refresh of an accepted trigger is ignored too; + * that with view tracking off a trigger view is not recorded, so nothing is sent; and that with + * both allowed a trigger view sends the queue and enters the zone once the server accepted it. + */ + @Test + public void journeyTriggerView_refreshesLikeAnEvent_neverWhileViewTrackingIsOff_andRczOffIgnoresEveryRefresh() throws Exception { + startQueueServer(); + server.settings.respondWith(new ServerConfigBuilder() + .journeyTriggerViews(names("checkout")) + .journeyTriggerEvents(names("purchase")) + .refreshContentZone(false)); + init(networkedContentConfig()); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(display); + + Countly.instance().views().startView("home"); + Countly.instance().content().refreshContentZone(); + TestUtils.validateEQSize(1); + Assert.assertEquals(1, logsContaining("refreshing the content zone is disabled by the SDK behavior settings")); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("purchase"); + awaitLogs("refreshing the content zone is disabled by the SDK behavior settings", 2); + awaitEmptyQueue(); + assertZoneIsInactive(0); + Assert.assertEquals("[\"[CLY]_view\",\"purchase\"]", eventKeys(queueServer.eventRequests().get(0)).toString()); + + push(new ServerConfigBuilder().refreshContentZone(true).viewTracking(false)); + Assert.assertNull(Countly.instance().views().startView("checkout")); + TestUtils.validateEQSize(0); + Assert.assertEquals(0, queuedRequestCount()); + assertZoneIsInactive(0); + + push(new ServerConfigBuilder().viewTracking(true)); + Countly.instance().views().startView("checkout"); + awaitZoneFetch(1); + List> eventRequests = queueServer.eventRequests(); + Assert.assertEquals(2, eventRequests.size()); + JSONObject view = new JSONArray(eventRequests.get(1).get("events")).getJSONObject(0); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, view.getString("key")); + Assert.assertEquals("checkout", view.getJSONObject("segmentation").getString(ModuleViews.KEY_NAME)); + awaitEmptyQueue(); + } + + // endregion + // region helpers + + /** + * Stands in for the settings and the content endpoints of the server. Settings requests go to a + * {@link ModuleConfigurationTests.ServerConfigResponder}; content fetches are recorded and answered + * on the calling thread with {@link #contentResponse}, unless networking is off, where the SDK's own + * request maker sends nothing. + */ + private static final class SettingsAndContentServer implements ImmediateRequestGenerator { + final ModuleConfigurationTests.ServerConfigResponder settings = new ModuleConfigurationTests.ServerConfigResponder(); + final List contentFetches = new CopyOnWriteArrayList<>(); + volatile String contentResponse = NO_CONTENT_RESPONSE; + + /** + * A request maker that routes by endpoint. + * + * @return the request maker + */ + @Override + public ImmediateRequestI createImmediateRequestMaker() { + ImmediateRequestI settingsRequestMaker = settings.createImmediateRequestMaker(); + return (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> { + if (!CONTENT_ENDPOINT.equals(customEndpoint)) { + settingsRequestMaker.doWork(requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log); + return; + } + if (!networkingIsEnabled) { + callback.callback(null); + return; + } + contentFetches.add(requestData); + callback.callback(new JSONObject(contentResponse)); + }; + } + } + + /** + * A local HTTP server standing in for the request queue endpoint {@code /i}: it records the + * parameters of every request and accepts it, rejects it with a 500, or drops the connection + * without any response. + */ + private static final class QueueServer { + static final int ACCEPT = 0; + static final int REJECT = 1; + static final int DROP = 2; + + final HttpServer http; + final List> received = new CopyOnWriteArrayList<>(); + volatile int mode = ACCEPT; + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + QueueServer() throws IOException { + http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + http.createContext("/", this::handle); + http.setExecutor(Executors.newSingleThreadExecutor()); + http.start(); + } + + /** + * Records a request and answers it as {@link #mode} says. + * + * @param exchange the request and its response + * @throws IOException when the response cannot be written + */ + private void handle(HttpExchange exchange) throws IOException { + String query = exchange.getRequestURI().getRawQuery(); + String body = new String(readAll(exchange.getRequestBody()), StandardCharsets.UTF_8); + received.add(parseParams(query != null && !query.isEmpty() ? query : body)); + + int currentMode = mode; + if (currentMode == DROP) { + exchange.close(); + return; + } + + byte[] response = (currentMode == ACCEPT ? "{\"result\":\"Success\"}" : "{\"result\":\"Server error\"}").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(currentMode == ACCEPT ? 200 : 500, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + + /** + * The received requests that carry events, in the order they arrived. + * + * @return the parameters of each request + */ + List> eventRequests() { + List> requests = new ArrayList<>(); + for (Map params : received) { + if (params.containsKey("events")) { + requests.add(params); + } + } + return requests; + } + + /** + * The port the server listens on. + * + * @return the port + */ + int port() { + return http.getAddress().getPort(); + } + } + + /** + * A display that records what it was asked to show and lets a test close it. + */ + private static final class FakeDisplay implements ContentDisplay { + final List presented = new CopyOnWriteArrayList<>(); + volatile ContentCloseCallback lastCallback; + + /** + * A landscape screen. + * + * @return the screen + */ + @Override + public ContentScreen getScreen() { + return new ContentScreen(1600, 900); + } + + /** + * Records the content and keeps the callback that closes it. + * + * @param content the content to show + * @param onClosed called when the content closes + */ + @Override + public void present(ContentData content, ContentCloseCallback onClosed) { + presented.add(content); + lastCallback = onClosed; + } + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests and content fetches. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.settings.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings test configuration with content and events, keeping every log line. + * + * @return the configuration + */ + private Config contentConfig() { + return TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Content, Config.Feature.Events) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * A configuration pointing at {@link #queueServer}, with content, events and views, an event + * queue that only a trigger flushes, no request cooldowns, and every log line kept. + * + * @return the configuration + */ + private Config networkedContentConfig() { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + return new Config("http://127.0.0.1:" + queueServer.port(), TestUtils.SERVER_APP_KEY, directory) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .enableFeatures(Config.Feature.Content, Config.Feature.Events, Config.Feature.Views) + .setEventQueueSizeToSend(100) + .setNetworkRequestCooldown(0) + .setNetworkImportantRequestCooldown(0) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * Starts {@link #queueServer}, accepting every request. + * + * @throws IOException when the server cannot start + */ + private void startQueueServer() throws IOException { + queueServer = new QueueServer(); + } + + /** + * Holds the first tick of every zone timer an hour away, so the zone fetches only when a test + * ticks it by hand. + */ + private static void holdTheZoneTimer() { + contentModule().startedAtForTests(System.currentTimeMillis() + 60L * 60L * 1000L); + } + + /** + * The content module of the running SDK. + * + * @return the module + */ + private static ModuleContent contentModule() { + return SDKCore.instance.module(ModuleContent.class); + } + + /** + * The timer of the active zone. + * + * @return the timer, {@code null} while no zone is active + */ + private static CountlyTimer zoneTimer() { + return contentModule().contentTimer; + } + + /** + * Ticks the zone once by hand and asserts the total number of content fetches after the tick. + * + * @param expectedFetches the fetches expected so far + */ + private void assertZoneFetches(int expectedFetches) { + contentModule().onZoneTimerTick(); + Assert.assertEquals(expectedFetches, server.contentFetches.size()); + } + + /** + * Ticks the zone once by hand and asserts that it fetched nothing, as no zone is active. + * + * @param fetchesSoFar the fetches made so far + */ + private void assertZoneIsInactive(int fetchesSoFar) { + assertZoneFetches(fetchesSoFar); + } + + /** + * Ticks the zone by hand until it fetches, which it does once a refresh entered it. + * + * @param expectedFetches the fetches expected once it fetched + */ + private void awaitZoneFetch(int expectedFetches) throws InterruptedException { + boolean fetched = waitFor(() -> { + contentModule().onZoneTimerTick(); + return server.contentFetches.size() >= expectedFetches; + }); + Assert.assertTrue("the content zone never fetched, logs: " + logs, fetched); + Assert.assertEquals(expectedFetches, server.contentFetches.size()); + } + + /** + * Waits until the log holds a number of lines containing a text. + * + * @param text the text + * @param count how many lines + */ + private void awaitLogs(String text, int count) throws InterruptedException { + Assert.assertTrue("never logged [" + text + "] " + count + " times", waitFor(() -> logsContaining(text) >= count)); + Assert.assertEquals(count, logsContaining(text)); + } + + /** + * Waits until every stored request was sent and removed. + */ + private void awaitEmptyQueue() throws InterruptedException { + Assert.assertTrue("the request queue never drained", waitFor(() -> queuedRequestCount() == 0)); + } + + /** + * How many log lines so far contain a text. + * + * @param text the text + * @return the number of lines + */ + private int logsContaining(String text) { + int count = 0; + for (String line : logs) { + if (line.contains(text)) { + count++; + } + } + return count; + } + + /** + * Counts the stored requests without reading them, as the network loop can remove one while it + * is read. + * + * @return the number of stored requests + */ + private static int queuedRequestCount() { + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith("[CLY]_request_")); + return files == null ? 0 : files.length; + } + + /** + * The storage IDs of the stored requests. + * + * @return the IDs + */ + private static Set storedRequestIds() { + Set ids = new HashSet<>(); + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith("[CLY]_request_")); + if (files != null) { + for (File file : files) { + ids.add(Long.parseLong(file.getName().substring("[CLY]_request_".length()))); + } + } + return ids; + } + + /** + * The keys of the events a request carries, in order. + * + * @param params the parameters of the request + * @return the keys as a JSON array + */ + private static JSONArray eventKeys(Map params) { + JSONArray events = new JSONArray(params.get("events")); + JSONArray keys = new JSONArray(); + for (int i = 0; i < events.length(); i++) { + keys.put(events.getJSONObject(i).getString("key")); + } + return keys; + } + + /** + * Polls until the condition holds or {@link #WAIT_BUDGET_MS} runs out, nudging the send loop on + * every round, as it only picks up a request when something is pushed. + * + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitFor(BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_BUDGET_MS; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + SDKCore core = SDKCore.instance; + if (core != null && core.networking != null) { + core.networking.check(core.config); + } + Thread.sleep(50); + } + return condition.getAsBoolean(); + } + + /** + * Decodes url encoded request parameters. + * + * @param data the parameters + * @return the parameters by name + */ + private static Map parseParams(String data) { + Map params = new HashMap<>(); + for (String pair : data.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return params; + } + + /** + * Reads a stream to its end. + * + * @param stream the stream + * @return the bytes + * @throws IOException when the stream cannot be read + */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationFiltersTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationFiltersTests.java new file mode 100644 index 00000000..cfdee0be --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationFiltersTests.java @@ -0,0 +1,457 @@ +package ly.count.sdk.java.internal; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.PredefinedUserPropertyKeys; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The listing filters and the user property cache limit of the SDK behavior settings, observed + * through the public API: which custom events, segmentation keys and custom user properties reach + * the event queue and the request queue, while internal events and predefined user properties pass. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread, so the response of the fetch made at init applies before init returns and a + * later one is pushed by fetching the way the refresh timer does. Networking is off through the + * settings, so every stored request stays on disk to be read. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationFiltersTests { + + private ModuleConfigurationTests.ServerConfigResponder server; + private final List warnings = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module and with a fresh responder. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ModuleConfigurationTests.ServerConfigResponder(); + warnings.clear(); + } + + /** + * Stops the SDK and clears its data. + */ + @After + public void afterTest() { + Countly.instance().halt(); + } + + // region scenarios + + /** + * {@code eb} and then {@code ew} decide which custom events are recorded, while internal events + * always pass. + *

+ * Verifies that with a blacklist, listed keys, spaces and dashes included, are dropped with one + * warning each, timed events too, while an unlisted key, a view and an internal event listed in + * the blacklist are recorded; that a response bringing a whitelist replaces the blacklist, so only + * its keys are recorded, a view still is and a key the blacklist dropped is dropped again; and that + * an empty whitelist allows every key. + */ + @Test + public void eventFilter_blacklistThenWhitelist_decideCustomEventsOnly_andAnEmptyListAllowsEverything() { + server.respondWith(new ServerConfigBuilder() + .eventFilterList(names("blocked_event", "event with spaces", "event-with-dashes", ModuleViews.KEY_VIEW_EVENT, "[CLY]_star_rating"), false) + .networking(false)); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Views).setEventQueueSizeToSend(100)); + + Countly.instance().events().recordEvent("blocked_event"); + Countly.instance().events().recordEvent("event with spaces"); + Countly.instance().events().recordEvent("event-with-dashes", TestUtils.map("colour", "red")); + Countly.instance().events().startEvent("blocked_event"); + Countly.instance().events().endEvent("blocked_event"); + TestUtils.validateEQSize(0); + Assert.assertEquals(4, warningsContaining("is filtered out by the event filter")); + + Countly.instance().events().recordEvent("allowed_event"); + Countly.instance().views().startView("home"); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Assert.assertEquals(Arrays.asList("allowed_event", ModuleViews.KEY_VIEW_EVENT, "[CLY]_star_rating"), keysInEQ()); + + push(new ServerConfigBuilder().eventFilterList(names("allowed_event", "another_allowed"), true)); + Countly.instance().events().recordEvent("allowed_event"); + Countly.instance().events().recordEvent("not_in_whitelist"); + Countly.instance().events().recordEvent("another_allowed", 2); + Countly.instance().events().recordEvent("blocked_event"); + Countly.instance().views().stopViewWithName("home"); + Assert.assertEquals(Arrays.asList("allowed_event", ModuleViews.KEY_VIEW_EVENT, "[CLY]_star_rating", + "allowed_event", "another_allowed", ModuleViews.KEY_VIEW_EVENT), keysInEQ()); + Assert.assertEquals(6, warningsContaining("is filtered out by the event filter")); + + push(new ServerConfigBuilder().eventFilterList(names(), true)); + Countly.instance().events().recordEvent("not_in_whitelist"); + Countly.instance().events().recordEvent("blocked_event"); + List keys = keysInEQ(); + Assert.assertEquals(Arrays.asList("not_in_whitelist", "blocked_event"), keys.subList(6, keys.size())); + Assert.assertEquals(6, warningsContaining("is filtered out by the event filter")); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + } + + /** + * {@code sb} and {@code esb}, then {@code sw} and {@code esw}, remove segmentation keys of custom + * events on a copy of the map the caller passed. + *

+ * Verifies on the wire that the segmentation blacklist and then the blacklist of each event apply + * together, that an event without rules keeps everything but the general blacklist, that the + * caller's unmodifiable map is left as it was and can be passed again, that an internal event and + * a view keep every key, that empty and missing segmentation still record, and that a response + * bringing whitelists replaces both blacklists, the general whitelist applying before the one of + * the event. + */ + @Test + public void segmentationFilters_generalThenPerEvent_filterACopyOfTheCallersMap_andNeverTouchInternalEvents() { + Map> perEvent = new LinkedHashMap<>(); + perEvent.put("event1", names("blocked_for_event1")); + perEvent.put("event2", names("key_b")); + server.respondWith(new ServerConfigBuilder() + .segmentationFilterList(names("general_blocked"), false) + .eventSegmentationFilterMap(perEvent, false) + .eventQueueSize(1) + .networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events, Config.Feature.Views)); + + Map callers = Collections.unmodifiableMap(TestUtils.map("general_blocked", "g", "blocked_for_event1", "e1", "key_a", "a", "key_b", "b")); + Countly.instance().events().recordEvent("event1", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event2", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event3", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("[CLY]_star_rating", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().views().startView("home", TestUtils.map("general_blocked", "v")); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event1", new HashMap<>()); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event1"); + TestUtils.letTheClockCatchUp(); + + List sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 7, sent.size()); + assertEvent(sent.get(0), "event1", TestUtils.map("key_a", "a", "key_b", "b")); + assertEvent(sent.get(1), "event2", TestUtils.map("blocked_for_event1", "e1", "key_a", "a")); + assertEvent(sent.get(2), "event3", TestUtils.map("blocked_for_event1", "e1", "key_a", "a", "key_b", "b")); + assertEvent(sent.get(3), "[CLY]_star_rating", callers); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, sent.get(4).key); + Assert.assertEquals("v", sent.get(4).segmentation.get("general_blocked")); + Assert.assertEquals("home", sent.get(4).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertEquals("event1", sent.get(5).key); + Assert.assertTrue(sent.get(5).segmentation == null || sent.get(5).segmentation.isEmpty()); + Assert.assertEquals("event1", sent.get(6).key); + Assert.assertTrue(sent.get(6).segmentation == null || sent.get(6).segmentation.isEmpty()); + Assert.assertEquals(TestUtils.map("general_blocked", "g", "blocked_for_event1", "e1", "key_a", "a", "key_b", "b"), callers); + + Map> perEventWhitelist = new LinkedHashMap<>(); + perEventWhitelist.put("event1", names("key_a")); + push(new ServerConfigBuilder() + .segmentationFilterList(names("key_a", "key_b", "blocked_for_event1"), true) + .eventSegmentationFilterMap(perEventWhitelist, true)); + Countly.instance().events().recordEvent("event1", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event2", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event4", TestUtils.map("key_b", "b", "other", "o")); + TestUtils.letTheClockCatchUp(); + + sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 10, sent.size()); + assertEvent(sent.get(7), "event1", TestUtils.map("key_a", "a")); + assertEvent(sent.get(8), "event2", TestUtils.map("blocked_for_event1", "e1", "key_a", "a", "key_b", "b")); + assertEvent(sent.get(9), "event4", TestUtils.map("key_b", "b")); + Assert.assertEquals(4, callers.size()); + } + + /** + * {@code upb} and then {@code upw} decide which custom user properties are set and modified, + * while predefined properties are never filtered. + *

+ * Verifies on the wire that with a blacklist listing custom keys and predefined ones, the + * predefined name and email are still sent while a listed custom key is dropped from a batch, + * from a single set and from each of the nine modifications, with one warning per dropped call, + * and every unlisted key keeps its set or modification; and that a response bringing a whitelist + * keeps only its custom keys while a predefined key outside it is still sent. + */ + @Test + public void userPropertyFilter_appliesToCustomKeysOfEverySetAndModification_whilePredefinedKeysBypassIt() { + server.respondWith(new ServerConfigBuilder() + .userPropertyFilterList(names("blocked_prop", "blocked_counter", PredefinedUserPropertyKeys.NAME, PredefinedUserPropertyKeys.EMAIL), false) + .networking(false)); + init(configWithWarnings()); + + Map batch = new LinkedHashMap<>(); + batch.put(PredefinedUserPropertyKeys.NAME, "John"); + batch.put(PredefinedUserPropertyKeys.EMAIL, "john@example.com"); + batch.put("blocked_prop", "value1"); + batch.put("allowed_prop", "value2"); + Countly.instance().userProfile().setProperties(Collections.unmodifiableMap(batch)); + Countly.instance().userProfile().setProperty("blocked_prop", "value3"); + modifyWithEveryOperation("blocked_counter"); + modifyWithEveryOperation("allowed_counter"); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = userDetails(0); + Assert.assertEquals("John", details.getString(PredefinedUserPropertyKeys.NAME)); + Assert.assertEquals("john@example.com", details.getString(PredefinedUserPropertyKeys.EMAIL)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("allowed_prop", "allowed_counter"), custom.keySet()); + Assert.assertEquals("value2", custom.getString("allowed_prop")); + JSONObject counter = custom.getJSONObject("allowed_counter"); + Assert.assertEquals(names("$inc", "$mul", "$max", "$min", "$setOnce", "$push", "$addToSet", "$pull"), counter.keySet()); + Assert.assertEquals(5, counter.getInt("$inc")); + Assert.assertEquals(11, warningsContaining("is filtered out by the user property filter")); + + push(new ServerConfigBuilder().userPropertyFilterList(names("plan", "visits"), true)); + Countly.instance().userProfile().setProperty("plan", "gold"); + Countly.instance().userProfile().setProperty("allowed_prop", "not_listed_now"); + Countly.instance().userProfile().setProperty(PredefinedUserPropertyKeys.USERNAME, "jdoe"); + Countly.instance().userProfile().increment("visits"); + Countly.instance().userProfile().increment("allowed_counter"); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + details = userDetails(1); + Assert.assertEquals("jdoe", details.getString(PredefinedUserPropertyKeys.USERNAME)); + custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("plan", "visits"), custom.keySet()); + Assert.assertEquals("gold", custom.getString("plan")); + Assert.assertEquals(1, custom.getJSONObject("visits").getInt("$inc")); + Assert.assertEquals(13, warningsContaining("is filtered out by the user property filter")); + } + + /** + * {@code upcl} caps the pending custom properties and the keys with pending modifications apart. + *

+ * Verifies that without the setting any number of custom properties is sent; that with it, a + * batch over the limit keeps its newest keys, predefined properties are neither counted nor + * dropped, updating a pending key does not count again, a new key drops the oldest one, and each + * drop warns; that modifications count distinct keys, so repeating one does not count, and going + * over the limit drops every modification of the oldest key; that the request carries exactly + * what was kept; and that a save starts the count again. + */ + @Test + public void userPropertyCacheLimit_capsPendingCustomSetsAndModifiedKeysApart_droppingTheOldestFirst() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(configWithWarnings()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getUserPropertyCacheLimit()); + + for (int i = 1; i <= 150; i++) { + Countly.instance().userProfile().setProperty("bulk_" + i, i); + } + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(150, userDetails(0).getJSONObject(ModuleUserProfile.CUSTOM_KEY).length()); + + push(new ServerConfigBuilder().userPropertyCacheLimit(3)); + Map batch = new LinkedHashMap<>(); + for (int i = 1; i <= 5; i++) { + batch.put("prop" + i, "value" + i); + } + batch.put(PredefinedUserPropertyKeys.NAME, "John"); + batch.put(PredefinedUserPropertyKeys.EMAIL, "john@example.com"); + batch.put(PredefinedUserPropertyKeys.USERNAME, "jdoe"); + Countly.instance().userProfile().setProperties(batch); + Assert.assertEquals(1, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().setProperty("prop3", "updated"); + Assert.assertEquals(1, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().setProperty("prop6", "value6"); + Assert.assertEquals(2, warningsContaining("over the cache limit of [3]")); + + Countly.instance().userProfile().incrementBy("counter1", 1); + Countly.instance().userProfile().incrementBy("counter2", 2); + Countly.instance().userProfile().incrementBy("counter1", 10); + Countly.instance().userProfile().incrementBy("counter3", 3); + Assert.assertEquals(2, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().multiply("factor", 2); + Countly.instance().userProfile().push("list", "x"); + Assert.assertEquals(4, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = userDetails(1); + Assert.assertEquals("John", details.getString(PredefinedUserPropertyKeys.NAME)); + Assert.assertEquals("john@example.com", details.getString(PredefinedUserPropertyKeys.EMAIL)); + Assert.assertEquals("jdoe", details.getString(PredefinedUserPropertyKeys.USERNAME)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("prop4", "prop5", "prop6", "counter3", "factor", "list"), custom.keySet()); + Assert.assertEquals("value4", custom.getString("prop4")); + Assert.assertEquals(3, custom.getJSONObject("counter3").getInt("$inc")); + Assert.assertEquals(2.0, custom.getJSONObject("factor").getDouble("$mul"), 0.0); + Assert.assertEquals("x", custom.getJSONObject("list").getString("$push")); + + Countly.instance().userProfile().setProperties(TestUtils.map("fresh1", "a", "fresh2", "b", "fresh3", "c")); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(3, userDetails(2).getJSONObject(ModuleUserProfile.CUSTOM_KEY).length()); + Assert.assertEquals(4, warningsContaining("over the cache limit of [3]")); + } + + // endregion + // region helpers + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings test configuration with a listener that keeps every warning. + * + * @return the configuration + */ + private Config configWithWarnings() { + return TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (level == Config.LoggingLevel.WARN) { + warnings.add(message); + } + }); + } + + /** + * How many warnings so far contain a text. + * + * @param text the text + * @return the number of warnings + */ + private int warningsContaining(String text) { + int count = 0; + for (String warning : warnings) { + if (warning.contains(text)) { + count++; + } + } + return count; + } + + /** + * The keys of the events in the event queue, in queue order. + * + * @return the keys + */ + private static List keysInEQ() { + List keys = new ArrayList<>(); + for (EventImpl event : TestUtils.getCurrentEQ()) { + keys.add(event.key); + } + return keys; + } + + /** + * Every event of every stored request, in queue order. + * + * @return the events + */ + private static List sentEvents() { + List events = new ArrayList<>(); + int requestCount = TestUtils.getCurrentRQ().length; + for (int i = 0; i < requestCount; i++) { + events.addAll(TestUtils.readEventsFromRequest(i, TestUtils.DEVICE_ID)); + } + return events; + } + + /** + * The keys of events, in order. + * + * @param events the events + * @return the keys + */ + private static List keysOf(List events) { + List keys = new ArrayList<>(); + for (EventImpl event : events) { + keys.add(event.key); + } + return keys; + } + + /** + * Asserts the key and the whole segmentation of an event. + * + * @param event the event + * @param key the expected key + * @param segmentation the expected segmentation + */ + private static void assertEvent(EventImpl event, String key, Map segmentation) { + Assert.assertEquals(key, event.key); + Assert.assertEquals(segmentation, new HashMap<>(event.segmentation)); + } + + /** + * The {@code user_details} object of a stored request. + * + * @param requestIndex the index of the request in the queue + * @return the user details + */ + private static JSONObject userDetails(int requestIndex) { + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertTrue("no request at index " + requestIndex + " in " + Arrays.toString(requests), requests.length > requestIndex); + return new JSONObject(requests[requestIndex].get("user_details")); + } + + /** + * Applies each of the nine modifications of the user profile interface to one key. + * + * @param key the custom user property key + */ + private static void modifyWithEveryOperation(String key) { + ModuleUserProfile.UserProfile profile = Countly.instance().userProfile(); + profile.increment(key); + profile.incrementBy(key, 4); + profile.multiply(key, 3); + profile.saveMax(key, 10); + profile.saveMin(key, 1); + profile.setOnce(key, "first"); + profile.push(key, "a"); + profile.pushUnique(key, "b"); + profile.pull(key, "c"); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationGatesTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationGatesTests.java new file mode 100644 index 00000000..5cd3b27a --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationGatesTests.java @@ -0,0 +1,820 @@ +package ly.count.sdk.java.internal; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.View; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * The switches and tunables of the SDK behavior settings, observed through the public API: what each + * one lets into the request queue and the event queue, both when it comes from the stored or provided + * settings at init and when a server response changes it while the SDK runs. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread: the response of the fetch made at init is applied before init returns, and a + * later one is pushed by fetching the way the refresh timer does. Most scenarios switch networking off + * through the settings, so every stored request stays on disk to be read. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationGatesTests { + + private ModuleConfigurationTests.ServerConfigResponder server; + private Thread.UncaughtExceptionHandler defaultHandlerBeforeTest; + + /** + * Starts every test from empty storage, without a test module and with a fresh responder. + */ + @Before + public void beforeTest() { + defaultHandlerBeforeTest = Thread.getDefaultUncaughtExceptionHandler(); + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ModuleConfigurationTests.ServerConfigResponder(); + } + + /** + * Stops the SDK, clearing its data, and puts back the default uncaught exception handler. + */ + @After + public void afterTest() { + Countly.instance().halt(); + SDKCore.testDummyModule = null; + Thread.setDefaultUncaughtExceptionHandler(defaultHandlerBeforeTest); + } + + // region scenarios + + /** + * {@code tracking} off keeps everything new out of storage. + *

+ * Verifies with the switch provided at init that custom and internal events, timed events, views, + * sessions, breadcrumbs, handled and unhandled crashes, location, user properties, an A/B test + * enrollment and a device ID merge store no request and no event and leave no crash file, that a + * crash file left by an earlier run is still turned into a request, and that events are recorded + * again once a response turns tracking back on. + */ + @Test + public void tracking_off_storesNoRequestAndNoEvent_whileACrashOfAnEarlierRunIsStillSent() { + long crashId = TimeUtils.uniqueTimestampMs(); + TestUtils.writeToFile("crash_" + crashId, new JSONObject().put("_error", "earlier_run_crash").put("_nonfatal", false).toString()); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events, Config.Feature.Sessions, Config.Feature.Views, Config.Feature.CrashReporting, + Config.Feature.Location, Config.Feature.UserProfiles, Config.Feature.RemoteConfig) + .setEventQueueSizeToSend(1) + .setSdkBehaviorSettings(new ServerConfigBuilder().tracking(false).networking(false).build())); + + Assert.assertFalse(provider().getTrackingEnabled()); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertTrue(requests[0].get("crash").contains("earlier_run_crash")); + Assert.assertEquals(0, filesWithPrefix("[CLY]_crash_")); + + Countly.instance().events().recordEvent("custom_event"); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Countly.instance().events().startEvent("timed_event"); + Countly.instance().events().endEvent("timed_event"); + Assert.assertNull(Countly.instance().views().startView("home")); + Countly.session().begin(); + Countly.session().update(); + Countly.session().end(); + Countly.instance().crashes().addCrashBreadcrumb("breadcrumb"); + Countly.instance().crashes().recordHandledException(new Exception("handled_while_tracking_is_off")); + Countly.instance().crashes().recordUnhandledException(new Exception("unhandled_while_tracking_is_off")); + Countly.instance().location().setLocation("TR", "Izmir", "1,2", "1.1.1.1"); + Countly.instance().location().disableLocation(); + Countly.instance().userProfile().setProperty("plan", "gold"); + Countly.instance().userProfile().save(); + Countly.instance().remoteConfig().enrollIntoABTestsForKeys(new String[] { "button_color" }); + Countly.instance().deviceId().changeWithMerge("merged_device_id"); + + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + TestUtils.validateEQSize(0); + Assert.assertEquals(0, filesWithPrefix("[CLY]_crash_")); + Assert.assertEquals(0, filesWithPrefix("[CLY]_session_")); + + push(new ServerConfigBuilder().tracking(true)); + Countly.instance().events().recordEvent("custom_event_after"); + + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(2, requests.length); + Assert.assertEquals("merged_device_id", requests[1].get("device_id")); + Assert.assertEquals("custom_event_after", new JSONArray(requests[1].get("events")).getJSONObject(0).getString("key")); + } + + /** + * {@code st} off ignores beginning, updating and ending a session, whatever calls them. + *

+ * Verifies that after a response turns session tracking off, an update, an end, a timer tick and a + * device ID change without merge send no session request while the running session keeps its + * state, that the session file left by that run is removed at the next init without an end + * request, that a session neither begins through the public API nor through a device ID change + * while the stored switch is off, that consent to sessions begins nothing either, and that a + * response turning it back on lets a session begin. + */ + @Test + public void sessionTracking_off_sendsNoSessionRequest_fromAnyCaller_andRecoveryDropsTheLeftoverSession() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Sessions)); + + Countly.session().begin(); + Assert.assertEquals(1, requestsWith("begin_session")); + + push(new ServerConfigBuilder().sessionTracking(false)); + Countly.session().update(); + Countly.session().end(); + SDKCore.instance.module(ModuleSessions.class).onTimer(); + Countly.instance().deviceId().changeWithoutMerge("device_id_2"); + + Assert.assertTrue(Countly.session().isActive()); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + Assert.assertEquals(0, requestsWith("session_duration") + requestsWith("end_session")); + Countly.instance().stop(); + Assert.assertEquals(1, filesWithPrefix("[CLY]_session_")); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Sessions)); + + Assert.assertFalse(provider().getSessionTrackingEnabled()); + Assert.assertEquals(0, filesWithPrefix("[CLY]_session_")); + SessionImpl session = (SessionImpl) Countly.session(); + session.begin(); + Assert.assertFalse(session.isActive()); + Assert.assertNull(session.update(null)); + Assert.assertNull(session.end(null, null, null)); + Countly.instance().deviceId().changeWithoutMerge("device_id_3"); + Assert.assertFalse(Countly.session().isActive()); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertEquals("1", requests[0].get("begin_session")); + Countly.instance().halt(); + + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Sessions) + .setRequiresConsent(true) + .setSdkBehaviorSettings(new ServerConfigBuilder().sessionTracking(false).networking(false).build())); + Countly.onConsent(Config.Feature.Sessions); + Assert.assertTrue(Countly.isTracking(Config.Feature.Sessions)); + Assert.assertFalse(Countly.session().isActive()); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().sessionTracking(true)); + Countly.session().begin(); + Assert.assertTrue(Countly.session().isActive()); + Assert.assertEquals(1, requestsWith("begin_session")); + } + + /** + * {@code vt} off ignores every view call, while custom events keep being recorded. + *

+ * Verifies that after a response turns view tracking off, starting, stopping by name and by ID, + * pausing, resuming and stopping all views record no {@code [CLY]_view} event, through the views + * interface and through the deprecated session views alike, that a custom event is still + * recorded, that the views keep running and end once a response turns view tracking back on, + * and that the stored switch keeps views off after a restart. + */ + @Test + public void viewTracking_off_ignoresEveryViewCall_whileCustomEventsStillRecord() { + Config config = TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.Sessions) + .setEventQueueSizeToSend(100); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(config); + + String homeId = Countly.instance().views().startView("home"); + String settingsId = Countly.instance().views().startView("settings"); + Assert.assertEquals(2, viewEvents().size()); + + push(new ServerConfigBuilder().viewTracking(false)); + Assert.assertNull(Countly.instance().views().startView("blocked")); + Assert.assertNull(Countly.instance().views().startAutoStoppedView("blocked_auto_stopped")); + Countly.instance().views().stopViewWithName("home"); + Countly.instance().views().stopViewWithID(settingsId); + Countly.instance().views().pauseViewWithID(homeId); + Countly.instance().views().resumeViewWithID(homeId); + Countly.instance().views().stopAllViews(null); + View legacyView = ((SessionImpl) Countly.session()).view("legacy_view"); + legacyView.stop(false); + Countly.instance().events().recordEvent("custom_event"); + + List events = TestUtils.getCurrentEQ(); + Assert.assertEquals(3, events.size()); + Assert.assertEquals("custom_event", events.get(2).key); + Assert.assertEquals(2, viewEvents().size()); + + push(new ServerConfigBuilder().viewTracking(true)); + Countly.instance().views().stopViewWithID(homeId); + Countly.instance().views().stopAllViews(null); + + List views = viewEvents(); + Assert.assertEquals(4, views.size()); + Assert.assertEquals("home", views.get(2).segmentation.get("name")); + Assert.assertEquals("settings", views.get(3).segmentation.get("name")); + Assert.assertNull(views.get(2).segmentation.get("visit")); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().viewTracking(false)); + Countly.instance().stop(); + server.pending(); + init(config); + + Assert.assertFalse(provider().getViewTrackingEnabled()); + Assert.assertNull(Countly.instance().views().startView("after_restart")); + Assert.assertEquals(4, viewEvents().size()); + Assert.assertEquals(5, TestUtils.getCurrentEQ().size()); + } + + /** + * {@code crt} and {@code acr} decide whether unhandled crashes are caught, a response can install + * the handler while the SDK runs, and handled exceptions follow {@code crt} alone. + *

+ * Verifies that the handler is not installed while {@code acr} is off, that a response turning it + * on installs it, that a later change never installs it twice, that it records an unhandled crash + * only while both switches are on at crash time and always hands the crash to the handler it + * replaced, that handled and unhandled exceptions reported through the crashes interface are + * recorded whenever {@code crt} is on, and that stopping the SDK leaves in place a handler the + * application installed over it, which then only reaches the application's handler. + */ + @Test + public void crashSwitches_decideWhatIsRecorded_andAResponseInstallsTheHandlerOnce() throws InterruptedException { + List reachedTheApp = new CopyOnWriteArrayList<>(); + Thread.UncaughtExceptionHandler appHandler = (thread, throwable) -> reachedTheApp.add(throwable); + Thread.setDefaultUncaughtExceptionHandler(appHandler); + List crashesBeingRecorded = new CopyOnWriteArrayList<>(); + + server.respondWith(new ServerConfigBuilder().automaticCrashReporting(false).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).setLogListener((message, level) -> { + if (message.startsWith("[ModuleCrash] onCrash:")) { + crashesBeingRecorded.add(message); + } + })); + + Assert.assertTrue(provider().getCrashReportingEnabled()); + Assert.assertFalse(provider().getAutomaticCrashReportingEnabled()); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + crashOnAThread(new RuntimeException("crash_while_acr_is_off")); + Assert.assertEquals(1, reachedTheApp.size()); + Assert.assertEquals(0, crashRequests().size()); + + push(new ServerConfigBuilder().automaticCrashReporting(true)); + Thread.UncaughtExceptionHandler installed = Thread.getDefaultUncaughtExceptionHandler(); + Assert.assertNotSame(appHandler, installed); + push(new ServerConfigBuilder().automaticCrashReporting(true).eventQueueSize(5)); + Assert.assertSame(installed, Thread.getDefaultUncaughtExceptionHandler()); + + crashOnAThread(new RuntimeException("crash_while_both_are_on")); + Assert.assertEquals(2, reachedTheApp.size()); + List crashes = crashRequests(); + Assert.assertEquals(1, crashes.size()); + Assert.assertTrue(crashes.get(0).getString("_error").contains("crash_while_both_are_on")); + Assert.assertFalse(crashes.get(0).getBoolean("_nonfatal")); + + push(new ServerConfigBuilder().crashReporting(false)); + crashOnAThread(new RuntimeException("crash_while_crt_is_off")); + Countly.instance().crashes().recordHandledException(new Exception("handled_while_crt_is_off")); + Assert.assertEquals(3, reachedTheApp.size()); + Assert.assertEquals(1, crashRequests().size()); + + push(new ServerConfigBuilder().crashReporting(true).automaticCrashReporting(false)); + crashOnAThread(new RuntimeException("crash_while_acr_is_off_again")); + Countly.instance().crashes().recordHandledException(new Exception("handled_while_crt_is_on")); + Countly.instance().crashes().recordUnhandledException(new Exception("reported_while_crt_is_on")); + Assert.assertEquals(4, reachedTheApp.size()); + Assert.assertSame(installed, Thread.getDefaultUncaughtExceptionHandler()); + crashes = crashRequests(); + Assert.assertEquals(3, crashes.size()); + Assert.assertTrue(crashes.get(1).getString("_error").contains("handled_while_crt_is_on")); + Assert.assertTrue(crashes.get(1).getBoolean("_nonfatal")); + Assert.assertTrue(crashes.get(2).getString("_error").contains("reported_while_crt_is_on")); + Assert.assertFalse(crashes.get(2).getBoolean("_nonfatal")); + Assert.assertEquals(3, crashesBeingRecorded.size()); + + List reachedTheLaterHandler = new CopyOnWriteArrayList<>(); + Thread.UncaughtExceptionHandler laterHandler = (thread, throwable) -> { + reachedTheLaterHandler.add(throwable); + installed.uncaughtException(thread, throwable); + }; + Thread.setDefaultUncaughtExceptionHandler(laterHandler); + push(new ServerConfigBuilder().automaticCrashReporting(true)); + Countly.instance().halt(); + + Assert.assertSame(laterHandler, Thread.getDefaultUncaughtExceptionHandler()); + crashOnAThread(new RuntimeException("crash_after_halt")); + Assert.assertEquals(1, reachedTheLaterHandler.size()); + Assert.assertEquals(5, reachedTheApp.size()); + Assert.assertEquals(3, crashesBeingRecorded.size()); + Assert.assertEquals(0, filesWithPrefix("[CLY]_crash_")); + Assert.assertEquals(0, filesWithPrefix("[CLY]_request_")); + } + + /** + * The handler installed at init follows the resolved {@code crt} and {@code acr} and crash consent. + *

+ * Verifies that the developer default installs it, that a developer who disabled unhandled crash + * reporting gets none unless the response of the fetch made at init turns {@code acr} on, which + * installs it once even though the response arrives before the crash module finished its init, + * that stored {@code crt} off keeps it out until a response turns {@code crt} on, that it waits + * for crash consent, and that each stop puts the application's handler back. + */ + @Test + public void crashHandler_atInit_followsTheResolvedSwitchesAndConsent() { + Thread.UncaughtExceptionHandler appHandler = (thread, throwable) -> { + }; + Thread.setDefaultUncaughtExceptionHandler(appHandler); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting)); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).disableUnhandledCrashReporting()); + Assert.assertFalse(provider().getAutomaticCrashReportingEnabled()); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + + server.respondWith(new ServerConfigBuilder().automaticCrashReporting(true)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).disableUnhandledCrashReporting()); + Assert.assertTrue(provider().getAutomaticCrashReportingEnabled()); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.CrashReporting) + .setSdkBehaviorSettings(new ServerConfigBuilder().crashReporting(false).build())); + Assert.assertTrue(provider().getAutomaticCrashReportingEnabled()); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + push(new ServerConfigBuilder().crashReporting(true)); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().stop(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).setRequiresConsent(true)); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.onConsent(Config.Feature.CrashReporting); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + } + + /** + * {@code lt} off keeps location out of every request, and turning it off erases the location + * stored on the server. + *

+ * Verifies that with location tracking on, a location is sent on its own and with the begin + * request; that a response turning it off sends one erase request and a later change sends no + * second one; that while it is off setting a location through the location interface, the + * deprecated session call and the deprecated request builder sends nothing and a begin request + * carries no location; that while session tracking is off a location is not held back for a + * session that will not begin; that a response at init turning it off while the developer + * disabled location sends a single erase request; and that the stored switch keeps the + * developer's location out of the next init. + */ + @Test + public void locationTracking_off_keepsLocationOutOfEveryRequest_andTurningItOffErasesIt() { + Config config = TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.Sessions); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(config); + + Countly.instance().location().setLocation("TR", "Izmir", "1,2", "1.1.1.1"); + Countly.session().begin(); + Countly.session().end(); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(3, requests.length); + Assert.assertEquals("Izmir", requests[0].get("city")); + Assert.assertEquals("1", requests[1].get("begin_session")); + Assert.assertEquals("Izmir", requests[1].get("city")); + Assert.assertEquals("1.1.1.1", requests[1].get("ip")); + + push(new ServerConfigBuilder().locationTracking(false)); + push(new ServerConfigBuilder().locationTracking(false).eventQueueSize(5)); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(4, requests.length); + Assert.assertEquals("", requests[3].get("location")); + Assert.assertFalse(requests[3].containsKey("city")); + + Countly.instance().location().setLocation("US", "New York", "3,4", "2.2.2.2"); + Countly.session().addLocation(5.0, 6.0); + Assert.assertNull(ModuleRequests.location(SDKCore.instance.config, 7.0, 8.0)); + Countly.session().begin(); + Countly.session().end(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(Arrays.toString(requests), 6, requests.length); + Assert.assertEquals("1", requests[4].get("begin_session")); + assertNoLocation(requests[4]); + + push(new ServerConfigBuilder().locationTracking(true).sessionTracking(false)); + Assert.assertNotNull(Countly.session()); + Countly.instance().location().setLocation("NL", "Amsterdam", null, null); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(7, requests.length); + Assert.assertEquals("Amsterdam", requests[6].get("city")); + Countly.instance().halt(); + + server.respondWith(new ServerConfigBuilder().locationTracking(false).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.Sessions).disableLocation()); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertEquals("", requests[0].get("location")); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.Sessions).setLocation("DE", "Berlin", "9,9", null)); + Assert.assertFalse(provider().getLocationTrackingEnabled()); + Countly.session().begin(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(2, requests.length); + Assert.assertEquals("1", requests[1].get("begin_session")); + assertNoLocation(requests[1]); + } + + /** + * {@code log} decides what the SDK prints, while the log listener hears every line. + *

+ * Verifies that with the developer level off, provided logging on prints every level from init + * and a response turning it off prints nothing; that a stored off silences a developer level of + * warnings until a response turns it on, which prints that level and not every level; and that + * without the setting the developer level prints as it is. The listener hears the lines that are + * not printed. + */ + @Test + public void logging_decidesWhatIsPrinted_whileTheListenerHearsEveryLine() { + List heard = new CopyOnWriteArrayList<>(); + ByteArrayOutputStream printed = new ByteArrayOutputStream(); + PrintStream originalOut = System.out; + System.setOut(new PrintStream(printed, true)); + try { + server.pending(); + init(configWithProbeListener(heard).setSdkBehaviorSettings(new ServerConfigBuilder().logging(true).build())); + Log log = SDKCore.instance.config.getLogger(); + Assert.assertEquals(Config.LoggingLevel.VERBOSE, log.getPrintLevel()); + log.v("probe_verbose_printed"); + Assert.assertTrue(printedText(printed).contains("probe_verbose_printed")); + + push(new ServerConfigBuilder().logging(false)); + Assert.assertEquals(Config.LoggingLevel.OFF, log.getPrintLevel()); + log.e("probe_error_silenced"); + Assert.assertFalse(printedText(printed).contains("probe_error_silenced")); + Assert.assertTrue(heard.contains("probe_error_silenced")); + Countly.instance().stop(); + + server.pending(); + init(configWithProbeListener(heard).setLoggingLevel(Config.LoggingLevel.WARN)); + log = SDKCore.instance.config.getLogger(); + Assert.assertEquals(Config.LoggingLevel.OFF, log.getPrintLevel()); + log.w("probe_warning_silenced"); + Assert.assertFalse(printedText(printed).contains("probe_warning_silenced")); + + push(new ServerConfigBuilder().logging(true)); + Assert.assertEquals(Config.LoggingLevel.WARN, log.getPrintLevel()); + log.d("probe_debug_below_the_level"); + log.w("probe_warning_printed"); + Assert.assertFalse(printedText(printed).contains("probe_debug_below_the_level")); + Assert.assertTrue(printedText(printed).contains("probe_warning_printed")); + Assert.assertTrue(heard.containsAll(Arrays.asList("probe_warning_silenced", "probe_debug_below_the_level", "probe_warning_printed"))); + Countly.instance().halt(); + + server.pending(); + init(configWithProbeListener(heard).setLoggingLevel(Config.LoggingLevel.ERROR)); + Assert.assertEquals(Config.LoggingLevel.ERROR, SDKCore.instance.config.getLogger().getPrintLevel()); + Countly.instance().halt(); + + init(configWithProbeListener(heard)); + Assert.assertEquals(Config.LoggingLevel.OFF, SDKCore.instance.config.getLogger().getPrintLevel()); + } finally { + System.setOut(originalOut); + } + } + + /** + * {@code rqs} caps the stored request queue only when the settings set it. + *

+ * Verifies that the developer request queue size leaves the stored queue unbounded, that once a + * response sets a size, storing a request drops the oldest ones over it with a warning saying how + * many, and that a stored size also holds at the next init for a crash of an earlier run that is + * turned into a request. + */ + @Test + public void requestQueueSize_fromTheSettings_capsTheStoredQueue_andWithoutItTheQueueIsUnbounded() { + List dropWarnings = new CopyOnWriteArrayList<>(); + Config config = TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setRequestQueueMaxSize(2) + .setLogListener((message, level) -> { + if (level == Config.LoggingLevel.WARN && message.startsWith("[ModuleRequests] dropOldestRequestsOverQueueLimit")) { + dropWarnings.add(message); + } + }); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(config); + + for (int i = 1; i <= 4; i++) { + Countly.instance().events().recordEvent("event_" + i); + letTheClockCatchUp(); + } + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(Arrays.asList("event_1", "event_2", "event_3", "event_4"), contentOfEachRequest()); + Assert.assertTrue(dropWarnings.isEmpty()); + + push(new ServerConfigBuilder().requestQueueSize(3)); + Countly.instance().events().recordEvent("event_5"); + Assert.assertEquals(Arrays.asList("event_3", "event_4", "event_5"), contentOfEachRequest()); + Assert.assertEquals(1, dropWarnings.size()); + Assert.assertTrue(dropWarnings.get(0), dropWarnings.get(0).contains("size of [3]") && dropWarnings.get(0).contains("dropped the [2] oldest")); + + Countly.instance().events().recordEvent("event_6"); + Assert.assertEquals(Arrays.asList("event_4", "event_5", "event_6"), contentOfEachRequest()); + Assert.assertEquals(2, dropWarnings.size()); + Assert.assertTrue(dropWarnings.get(1), dropWarnings.get(1).contains("dropped the [1] oldest")); + Countly.instance().stop(); + + TestUtils.writeToFile("crash_" + TimeUtils.uniqueTimestampMs(), new JSONObject().put("_error", "earlier_run_crash").put("_nonfatal", false).toString()); + server.pending(); + init(config); + Assert.assertTrue(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(Arrays.asList("event_5", "event_6", "crash"), contentOfEachRequest()); + Assert.assertEquals(3, dropWarnings.size()); + } + + /** + * {@code lbc} is read each time a breadcrumb is added. + *

+ * Verifies that the limit of the response at init overrides the developer's, that the crash + * request carries only the newest breadcrumbs within it, and that after a response shrinks the + * limit while breadcrumbs are kept, the next breadcrumb brings them within the new limit. + */ + @Test + public void breadcrumbLimit_isReadWhenABreadcrumbIsAdded() { + server.respondWith(new ServerConfigBuilder().breadcrumbLimit(3).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).setMaxBreadcrumbCount(10)); + Assert.assertEquals(3, provider().getMaxBreadcrumbCount()); + + addBreadcrumbs("a", 5); + Countly.instance().crashes().recordHandledException(new Exception("first")); + + push(new ServerConfigBuilder().breadcrumbLimit(5)); + addBreadcrumbs("b", 5); + push(new ServerConfigBuilder().breadcrumbLimit(2)); + Countly.instance().crashes().addCrashBreadcrumb("c1"); + Countly.instance().crashes().recordHandledException(new Exception("second")); + + List crashes = crashRequests(); + Assert.assertEquals(2, crashes.size()); + Assert.assertEquals("a3\na4\na5", crashes.get(0).getString("_logs")); + Assert.assertEquals("b5\nc1", crashes.get(1).getString("_logs")); + } + + /** + * {@code eqs} and {@code sui} changed by a response apply while the SDK runs. + *

+ * Verifies that the event queue is flushed at the new threshold the next time an event is added, + * that a changed session update interval restarts the global timer at that interval, and that a + * response repeating it keeps the running timer. + */ + @Test + public void eventQueueSizeAndSessionUpdateInterval_changedByAResponse_applyWhileTheSdkRuns() { + server.respondWith(new ServerConfigBuilder().eventQueueSize(5).sessionUpdateInterval(30).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events).setEventQueueSizeToSend(100)); + CountlyTimer timerAtInit = SDKCore.instance.countlyTimer; + Assert.assertEquals(30, timerAtInit.getTimerDelaySeconds()); + + Countly.instance().events().recordEvent("e1"); + Countly.instance().events().recordEvent("e2"); + push(new ServerConfigBuilder().eventQueueSize(3)); + TestUtils.validateEQSize(2); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + Countly.instance().events().recordEvent("e3"); + TestUtils.validateEQSize(0); + Assert.assertEquals(3, TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID).size()); + + push(new ServerConfigBuilder().eventQueueSize(1)); + Countly.instance().events().recordEvent("e4"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Assert.assertEquals("e4", TestUtils.readEventsFromRequest(1, TestUtils.DEVICE_ID).get(0).key); + + push(new ServerConfigBuilder().sessionUpdateInterval(45)); + CountlyTimer restarted = SDKCore.instance.countlyTimer; + Assert.assertNotSame(timerAtInit, restarted); + Assert.assertEquals(45, restarted.getTimerDelaySeconds()); + + push(new ServerConfigBuilder().sessionUpdateInterval(45).eventQueueSize(2)); + Assert.assertSame(restarted, SDKCore.instance.countlyTimer); + Countly.instance().events().recordEvent("e5"); + TestUtils.validateEQSize(1); + Countly.instance().events().recordEvent("e6"); + TestUtils.validateEQSize(0); + Assert.assertEquals(3, TestUtils.getCurrentRQ().length); + } + + // endregion + // region helpers + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + letTheClockCatchUp(); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + letTheClockCatchUp(); + server.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * Waits until the clock is past every timestamp handed out so far. A request file is named after + * a {@link TimeUtils#uniqueTimestampMs()} value, which is unique among the last ten values only, + * so a long burst of requests could reuse a name and overwrite a queued request. + */ + private static void letTheClockCatchUp() { + long latest = TimeUtils.uniqueTimestampMs(); + while (System.currentTimeMillis() <= latest) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** + * How many stored requests carry a parameter. + * + * @param param the parameter name + * @return the number of requests + */ + private static int requestsWith(String param) { + int count = 0; + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey(param)) { + count++; + } + } + return count; + } + + /** + * What each stored request carries, in queue order: the key of its first event, or + * {@code "crash"} for a crash request. + * + * @return one entry per request + */ + private static List contentOfEachRequest() { + List contents = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("crash")) { + contents.add("crash"); + } else { + contents.add(new JSONArray(request.get("events")).getJSONObject(0).getString("key")); + } + } + return contents; + } + + /** + * The crash objects of the stored crash requests, in queue order. + * + * @return the crash objects + */ + private static List crashRequests() { + List crashes = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("crash")) { + crashes.add(new JSONObject(request.get("crash"))); + } + } + return crashes; + } + + /** + * The {@code [CLY]_view} events in the event queue, in queue order. + * + * @return the view events + */ + private static List viewEvents() { + List views = new ArrayList<>(); + for (EventImpl event : TestUtils.getCurrentEQ()) { + if (ModuleViews.KEY_VIEW_EVENT.equals(event.key)) { + views.add(event); + } + } + return views; + } + + /** + * How many files in the storage directory start with a prefix. + * + * @param prefix the file name prefix + * @return the number of files + */ + private static int filesWithPrefix(String prefix) { + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith(prefix)); + return files == null ? 0 : files.length; + } + + /** + * Asserts that a request carries no location parameter, not even the empty one that erases it. + * + * @param request the request parameters + */ + private static void assertNoLocation(Map request) { + for (String param : Arrays.asList("location", "city", "country_code", "ip")) { + Assert.assertFalse("carries " + param + ": " + request, request.containsKey(param)); + } + } + + /** + * Adds numbered breadcrumbs, {@code prefix1} to {@code prefixN}. + * + * @param prefix the text before the number + * @param count how many + */ + private static void addBreadcrumbs(String prefix, int count) { + for (int i = 1; i <= count; i++) { + Countly.instance().crashes().addCrashBreadcrumb(prefix + i); + } + } + + /** + * Throws on a new thread and waits for it to end, which is after the default uncaught exception + * handler returned. + * + * @param crash what the thread throws + */ + private static void crashOnAThread(RuntimeException crash) throws InterruptedException { + Thread thread = new Thread(() -> { + throw crash; + }); + thread.start(); + thread.join(); + } + + /** + * The settings test configuration with a log listener that keeps the probe lines of the logging test. + * + * @param heard where the probe lines go + * @return the configuration + */ + private static Config configWithProbeListener(List heard) { + return TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (message.startsWith("probe_")) { + heard.add(message); + } + }); + } + + /** + * What was printed so far. + * + * @param printed the captured output + * @return the text + */ + private static String printedText(ByteArrayOutputStream printed) { + return new String(printed.toByteArray(), StandardCharsets.UTF_8); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationLimitsTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationLimitsTests.java new file mode 100644 index 00000000..55ec397d --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationLimitsTests.java @@ -0,0 +1,754 @@ +package ly.count.sdk.java.internal; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.Crash; +import ly.count.sdk.java.CrashProcessor; +import ly.count.sdk.java.PredefinedUserPropertyKeys; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The SDK internal limits of the SDK behavior settings, {@code lkl}, {@code lvs}, {@code lsv}, + * {@code ltlpt} and {@code ltl}, with the breadcrumb limit {@code lbc}, observed through the public + * API: what custom events, timed events, views, user properties, crashes and feedback widget results + * send once the settings set limits, how a later response changes them, and that nothing is cut + * while the settings set none. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread, so the response of the fetch made at init applies before init returns and a + * later one is pushed by fetching the way the refresh timer does. Networking is off through the + * settings wherever they are fetched, so every stored request stays on disk to be read. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationLimitsTests { + + private ModuleConfigurationTests.ServerConfigResponder server; + private final List warnings = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module, with a fresh responder and with + * the crash processor passing crashes through. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ModuleConfigurationTests.ServerConfigResponder(); + warnings.clear(); + DumpingCrashProcessor.replaceWithDump = false; + DumpingCrashProcessor.framesPerThread = 5; + } + + /** + * Stops the SDK and clears its data. + */ + @After + public void afterTest() { + Countly.instance().halt(); + DumpingCrashProcessor.replaceWithDump = false; + } + + // region scenarios + + /** + * {@code lkl}, {@code lvs} and {@code lsv} cut custom events after the listing filters and the + * journey trigger match, which see the full key, while an internal event only gets the string + * values of its widget result cut. + *

+ * Verifies on the wire that a journey trigger listed by its full key sends the queue at once, that + * its key is cut to 5 characters, that a segmentation key blacklisted by its full name and an + * entry of an unsupported type are removed before the entry limit counts while a key blacklisted + * only by its truncated form is kept, that of two keys cut to the same key the one that sorts last + * keeps its value, that string values are cut to 6 characters, that past 2 entries the keys that + * sort last are dropped and that the caller's map is left as it was; that an event whose truncated + * key is whitelisted but whose full + * key is not is dropped; that a timed event is cut the same way; and that a feedback widget + * result keeps every key and entry and the values the SDK adds, only its own strings being cut. + */ + @Test + public void customEvents_areCutAfterTheFiltersAndTheJourneyTriggerSawTheFullKey_andInternalEventsOnlyGetWidgetValuesCut() { + server.respondWith(smallLimits() + .eventFilterList(names("purchase_completed", "timed_checkout", "check"), true) + .segmentationFilterList(names("internal_notes", "payme"), false) + .journeyTriggerEvents(names("purchase_completed"))); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Feedback).setEventQueueSizeToSend(100)); + + Map segmentation = new HashMap<>(); + segmentation.put("payment_method", "credit_card"); + segmentation.put("category_a", "books"); + segmentation.put("category_b", "movies"); + segmentation.put("internal_notes", "removed_by_the_filter"); + segmentation.put("aa_unsupported", new Object()); + segmentation.put("zz_note", "dropped_by_the_entry_limit"); + Map callers = Collections.unmodifiableMap(new HashMap<>(segmentation)); + Countly.instance().events().recordEvent("purchase_completed", callers, 2, 9.99); + TestUtils.letTheClockCatchUp(); + + List sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 1, sent.size()); + assertEvent(sent.get(0), "purch", TestUtils.map("categ", "movies", "payme", "credit")); + Assert.assertEquals(2, sent.get(0).count); + Assert.assertEquals(9.99, sent.get(0).sum, 0.0); + Assert.assertEquals(segmentation, callers); + Assert.assertEquals(1, warningsContaining("[ModuleEvents] recordEventInternal, the key [purchase_completed] is longer than the key length limit of [5]")); + Assert.assertEquals(1, warningsContaining("its value replaces that of [categ]")); + Assert.assertEquals(1, warningsContaining("the segmentation has [3] entries, over the limit of [2]")); + + Countly.instance().events().recordEvent("checkout_started"); + Assert.assertEquals(1, warningsContaining("is filtered out by the event filter")); + + Countly.instance().events().startEvent("timed_checkout"); + Countly.instance().events().endEvent("timed_checkout", TestUtils.map("duration_bucket", "long_running", "attempt", 2), 1, null); + + CountlyFeedbackWidget widget = new CountlyFeedbackWidget(); + widget.widgetId = "rating_widget_identifier"; + widget.type = FeedbackWidgetType.rating; + widget.name = "rating"; + widget.tags = new String[0]; + Map widgetResult = new HashMap<>(); + widgetResult.put("rating", 4); + widgetResult.put("comment", "a comment longer than six characters"); + widgetResult.put("email", "user@example.com"); + widgetResult.put("contactMe", true); + Countly.instance().feedback().reportFeedbackWidgetManually(widget, null, widgetResult); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 2, queued.size()); + assertEvent(queued.get(0), "timed", TestUtils.map("attem", 2, "durat", "long_r")); + assertEvent(queued.get(1), FeedbackWidgetType.rating.eventKey, TestUtils.map("platform", WidgetUrlBuilder.PLATFORM, + "app_version", TestUtils.APPLICATION_VERSION, "widget_id", "rating_widget_identifier", + "rating", 4, "comment", "a comm", "email", "user@e", "contactMe", true)); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + } + + /** + * {@code lkl}, {@code lvs} and {@code lsv} cut the name and the segmentation of views, while the + * keys the SDK adds are neither cut nor counted. + *

+ * Verifies on the wire that the name is cut to 5 characters on the start and the end event, that + * the global, the added and the given segmentation are merged and then cut, that a key cut to a + * reserved key is removed so an end event never carries {@code visit}, that an entry of an + * unsupported type takes no place under the entry limit, that past 2 entries the keys that sort + * last are dropped while {@code name}, {@code visit}, {@code start} and + * {@code segment} stay with their values uncut, that a journey trigger view matches the name as + * sent, and that a view started under a long name is still stopped by that name. + */ + @Test + public void views_nameAndSegmentationAreCut_whileTheKeysTheSdkAddsAreNeitherCutNorCounted() { + server.respondWith(smallLimits().journeyTriggerViews(names("Setti"))); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Views).setEventQueueSizeToSend(100)); + String platform = SDKCore.instance.config.getSdkPlatform(); + Countly.instance().views().setGlobalViewSegmentation(TestUtils.map("global_key", "global_value")); + + String viewId = Countly.instance().views().startView("SettingsScreen", TestUtils.map("visitor_type", "returning", "aa_first", 1)); + TestUtils.letTheClockCatchUp(); + Countly.instance().views().addSegmentationToViewWithID(viewId, TestUtils.map("added_key", "added_value", "bb_second", 2)); + Countly.instance().views().stopViewWithID(viewId, TestUtils.map("visitor_type", "new_visitor", "cc_third", "third_value", "aa_unsupported", new int[] { 1 })); + TestUtils.letTheClockCatchUp(); + + List sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 2, sent.size()); + assertEvent(sent.get(0), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("aa_fi", 1, "globa", "global", + ModuleViews.KEY_NAME, "Setti", ModuleViews.KEY_VISIT, "1", ModuleViews.KEY_START, "1", ModuleViews.KEY_SEGMENT, platform)); + assertEvent(sent.get(1), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("added", "added_", "bb_se", 2, + ModuleViews.KEY_NAME, "Setti", ModuleViews.KEY_SEGMENT, platform)); + Assert.assertEquals(viewId, sent.get(1).id); + Assert.assertEquals(2, warningsContaining("[ModuleViews] createViewEventSegmentation, a key truncated to the reserved key [visit] is removed")); + + Countly.instance().views().startView("Home"); + Countly.instance().views().startView("ProfileScreen"); + Countly.instance().views().stopViewWithName("ProfileScreen"); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 3, queued.size()); + assertEvent(queued.get(0), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("globa", "global", + ModuleViews.KEY_NAME, "Home", ModuleViews.KEY_VISIT, "1", ModuleViews.KEY_SEGMENT, platform)); + Assert.assertEquals("Profi", queued.get(1).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertEquals("1", queued.get(1).segmentation.get(ModuleViews.KEY_VISIT)); + assertEvent(queued.get(2), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("globa", "global", + ModuleViews.KEY_NAME, "Profi", ModuleViews.KEY_SEGMENT, platform)); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + } + + /** + * {@code lkl}, {@code lvs} and {@code lsv} cut custom user properties, sets and modifications, + * while predefined properties only get their string values cut and the picture is never cut. The + * limits in effect when the request is built apply. + *

+ * Verifies on the wire that predefined keys stay whole, that the name, email and organization + * are cut to 6 characters while the picture URL, the gender and the birth year are sent as they + * are; that custom keys are cut to 5 characters, the key that sorts last winning a collision, + * string values cut to 6, and past 2 sets the keys that sort last are dropped; that modification + * keys and string operands are cut the same way and never count against the entry limit; and + * that a property set before a response lowered the key length limit is cut by the new limit. + */ + @Test + public void userProperties_customKeysAndValuesAreCutForSetsAndModifications_predefinedOnlyTheirValuesAndNeverThePicture() { + server.respondWith(smallLimits()); + init(configWithWarnings()); + + String pictureUrl = "https://example.com/pictures/avatar.png"; + Map batch = new LinkedHashMap<>(); + batch.put(PredefinedUserPropertyKeys.NAME, "Johnathan Doe"); + batch.put(PredefinedUserPropertyKeys.EMAIL, "johnathan@example.com"); + batch.put(PredefinedUserPropertyKeys.ORGANIZATION, "Countly Ltd"); + batch.put(PredefinedUserPropertyKeys.PICTURE_PATH, pictureUrl); + batch.put(PredefinedUserPropertyKeys.GENDER, "M"); + batch.put(PredefinedUserPropertyKeys.BIRTH_YEAR, 1990); + batch.put("favourite_colour", "turquoise"); + batch.put("favourite_food", "spaghetti"); + batch.put("aaa_first", 1); + batch.put("zzz_last", "dropped"); + Countly.instance().userProfile().setProperties(batch); + Countly.instance().userProfile().increment("login_count_total"); + Countly.instance().userProfile().push("visited_pages", "homepage_long"); + Countly.instance().userProfile().setOnce("first_seen_at", "2024-01-01"); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = userDetails(0); + Assert.assertEquals("Johnat", details.getString(PredefinedUserPropertyKeys.NAME)); + Assert.assertEquals("johnat", details.getString(PredefinedUserPropertyKeys.EMAIL)); + Assert.assertEquals("Countl", details.getString(PredefinedUserPropertyKeys.ORGANIZATION)); + Assert.assertEquals(pictureUrl, details.getString(PredefinedUserPropertyKeys.PICTURE)); + Assert.assertEquals("M", details.getString(PredefinedUserPropertyKeys.GENDER)); + Assert.assertEquals(1990, details.getInt(PredefinedUserPropertyKeys.BIRTH_YEAR)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("aaa_f", "favou", "login", "visit", "first"), custom.keySet()); + Assert.assertEquals(1, custom.getInt("aaa_f")); + Assert.assertEquals("spaghe", custom.getString("favou")); + Assert.assertEquals(1, custom.getJSONObject("login").getInt("$inc")); + Assert.assertEquals("homepa", custom.getJSONObject("visit").getString("$push")); + Assert.assertEquals("2024-0", custom.getJSONObject("first").getString("$setOnce")); + Assert.assertEquals(1, warningsContaining("[ModuleUserProfile] perform, the segmentation has [3] entries, over the limit of [2]")); + + Countly.instance().userProfile().setProperty("abc_property", "value"); + Countly.instance().userProfile().incrementBy("xyz_counter", 3); + push(new ServerConfigBuilder().keyLengthLimit(3)); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + custom = userDetails(1).getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("abc", "xyz"), custom.keySet()); + Assert.assertEquals("value", custom.getString("abc")); + Assert.assertEquals(3, custom.getJSONObject("xyz").getInt("$inc")); + } + + /** + * {@code ltl} cuts every stack trace line, {@code ltlpt} the lines of each thread of a trace dump, + * {@code lkl}, {@code lvs} and {@code lsv} the custom segments, and {@code lvs} and {@code lbc} + * the breadcrumbs, including what a crash processor adds. + *

+ * Verifies on the wire that a handled exception keeps every line of its stack trace, each cut to + * 20 characters, that its segments are cut to 5 character keys and 6 character values with the 2 + * keys that sort first kept, and that only the 2 newest breadcrumbs are sent, cut to 6 characters; + * and that when a crash processor replaces the stack trace with a dump of two threads and sets its + * own segments and breadcrumbs, each thread keeps its 3 top lines, the lines are cut to 20 + * characters and the segments and breadcrumbs it set are cut as well. + */ + @Test + public void crashes_stackTraceLinesSegmentsAndBreadcrumbsAreCut_includingWhatACrashProcessorAdds() { + server.respondWith(smallLimits().breadcrumbLimit(2)); + init(configWithWarnings().enableFeatures(Config.Feature.CrashReporting).setCrashProcessorClass(DumpingCrashProcessor.class)); + + Countly.instance().crashes().addCrashBreadcrumb("first_breadcrumb"); + Countly.instance().crashes().addCrashBreadcrumb("second_breadcrumb"); + Countly.instance().crashes().addCrashBreadcrumb("third_breadcrumb"); + Exception exception = new Exception("a message longer than twenty characters"); + Countly.instance().crashes().recordHandledException(exception, TestUtils.map("screen_name", "checkout_page", "retry_count", 3, "zz_last", "value")); + TestUtils.letTheClockCatchUp(); + + DumpingCrashProcessor.replaceWithDump = true; + Countly.instance().crashes().recordUnhandledException(new RuntimeException("dumped")); + TestUtils.letTheClockCatchUp(); + + List crashes = crashRequests(); + Assert.assertEquals(2, crashes.size()); + + JSONObject handled = crashes.get(0); + String fullStackTrace = stackTraceOf(exception); + Assert.assertTrue(lineCount(fullStackTrace) > 3); + Assert.assertEquals(truncateLines(fullStackTrace, 20), handled.getString("_error")); + Assert.assertEquals(lineCount(fullStackTrace), lineCount(handled.getString("_error"))); + Assert.assertTrue(handled.getBoolean("_nonfatal")); + Assert.assertEquals("second\nthird_", handled.getString("_logs")); + JSONObject segments = handled.getJSONObject("_custom"); + Assert.assertEquals(names("retry", "scree"), segments.keySet()); + Assert.assertEquals(3, segments.getInt("retry")); + Assert.assertEquals("checko", segments.getString("scree")); + + JSONObject dumped = crashes.get(1); + String frameLine = "\tat ly.count.Example\n"; + String expectedDump = "Thread [main]:" + System.lineSeparator() + frameLine + frameLine + frameLine + "\n\n" + + "Thread [worker-threa\n" + frameLine + frameLine + frameLine + "\n\n"; + Assert.assertEquals("anr", dumped.getString("_type")); + Assert.assertFalse(dumped.getBoolean("_nonfatal")); + Assert.assertEquals(expectedDump, dumped.getString("_error")); + segments = dumped.getJSONObject("_custom"); + Assert.assertEquals(names("anoth", "proce"), segments.keySet()); + Assert.assertEquals("anothe", segments.getString("anoth")); + Assert.assertEquals("proces", segments.getString("proce")); + Assert.assertEquals("proces", dumped.getString("_logs")); + Assert.assertEquals(2, warningsContaining("[ModuleCrash] onCrash, the stack trace of the thread")); + } + + /** + * A response that changes a limit applies from the next call, lowering or raising it. + *

+ * Verifies that an event recorded before any limit is sent whole, that after a response brings + * key and value limits the same call is cut by them, that a later response lowering the key + * length and adding an entry limit is merged over the value limit it leaves out, that one raising + * every limit sends the event whole again from the caller's untouched map, and that breadcrumbs + * and stack trace lines follow the limits in effect when each breadcrumb is added and when the + * crash is recorded. + */ + @Test + public void aResponseChangingTheLimits_appliesFromTheNextCall() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.CrashReporting).setEventQueueSizeToSend(100)); + Map segmentation = TestUtils.map("segment_key", "segment_value", "second", 2); + + Countly.instance().events().recordEvent("event_key", segmentation); + push(new ServerConfigBuilder().keyLengthLimit(8).valueSizeLimit(4)); + Countly.instance().events().recordEvent("event_key", segmentation); + push(new ServerConfigBuilder().keyLengthLimit(3).segmentationValuesLimit(1)); + Assert.assertEquals(4, provider().getMaxValueSize()); + Countly.instance().events().recordEvent("event_key", segmentation); + push(new ServerConfigBuilder().keyLengthLimit(64).valueSizeLimit(64).segmentationValuesLimit(64)); + Countly.instance().events().recordEvent("event_key", segmentation); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 4, queued.size()); + assertEvent(queued.get(0), "event_key", segmentation); + assertEvent(queued.get(1), "event_ke", TestUtils.map("segment_", "segm", "second", 2)); + assertEvent(queued.get(2), "eve", TestUtils.map("sec", 2)); + assertEvent(queued.get(3), "event_key", segmentation); + + push(new ServerConfigBuilder().valueSizeLimit(4)); + Countly.instance().crashes().addCrashBreadcrumb("alpha_breadcrumb"); + push(new ServerConfigBuilder().breadcrumbLimit(1).traceLengthLimit(12)); + Countly.instance().crashes().addCrashBreadcrumb("omega_breadcrumb"); + Exception exception = new Exception("recorded after the change"); + Countly.instance().crashes().recordHandledException(exception); + TestUtils.letTheClockCatchUp(); + + List crashes = crashRequests(); + Assert.assertEquals(1, crashes.size()); + Assert.assertEquals("omeg", crashes.get(0).getString("_logs")); + Assert.assertEquals(truncateLines(stackTraceOf(exception), 12), crashes.get(0).getString("_error")); + } + + /** + * With a live response that sets no limit, nothing any feature sends is cut, however long or wide. + *

+ * Verifies that the provider reports no key, value, entry, lines per thread and line length limit + * and the developer breadcrumb limit, and then the same as + * {@link #assertNothingIsCut()}. + */ + @Test + public void withSettingsThatSetNoLimit_nothingIsCut() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.CrashReporting, Config.Feature.Feedback) + .setEventQueueSizeToSend(1000).setCrashProcessorClass(DumpingCrashProcessor.class)); + + ConfigurationProvider provider = provider(); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxKeyLength()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxValueSize()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxSegmentationValues()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxStackTraceLinesPerThread()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxStackTraceLineLength()); + Assert.assertEquals(100, provider.getMaxBreadcrumbCount()); + + assertNothingIsCut(); + } + + /** + * Without any SDK behavior settings, the settings requests being disabled as for an upgrading app + * that never received any, nothing any feature sends is cut, however long or wide. + *

+ * Verifies the same as {@link #assertNothingIsCut()}. + */ + @Test + public void withoutSettings_nothingIsCut() { + Countly.instance().init(TestUtils.getBaseConfig().setLogListener(this::keepWarning) + .enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.CrashReporting, Config.Feature.Feedback) + .setEventQueueSizeToSend(1000).setCrashProcessorClass(DumpingCrashProcessor.class)); + + assertNothingIsCut(); + } + + // endregion + // region helpers + + /** + * Records a 200 character event key with 150 segments of 200 character keys and 300 character + * values as an event, a timed event and a view with its end event, reports a feedback widget + * result with a 300 character comment, sets 150 such custom user properties with a 300 character + * name and a modification, and records a crash with that segmentation, a 300 character message + * and breadcrumb and a crash processor dump of 40 lines per thread. Asserts that every key, value, + * entry and stack trace line is sent whole and that no limit warning was logged. + */ + private void assertNothingIsCut() { + String longKey = repeat("key_", 50); + String longValue = repeat("value_", 50); + Map wide = new HashMap<>(); + for (int i = 0; i < 150; i++) { + wide.put(longKey + i, longValue + i); + } + + Countly.instance().events().recordEvent(longKey, wide); + Countly.instance().events().startEvent(longKey + "timed"); + Countly.instance().events().endEvent(longKey + "timed", wide, 1, null); + String viewId = Countly.instance().views().startView(longKey, wide); + Countly.instance().views().stopViewWithID(viewId, wide); + CountlyFeedbackWidget widget = new CountlyFeedbackWidget(); + widget.widgetId = "rating_widget_identifier"; + widget.type = FeedbackWidgetType.rating; + widget.name = "rating"; + widget.tags = new String[0]; + Countly.instance().feedback().reportFeedbackWidgetManually(widget, null, TestUtils.map("rating", 5, "comment", longValue)); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 5, queued.size()); + assertEvent(queued.get(0), longKey, wide); + assertEvent(queued.get(1), longKey + "timed", wide); + Map viewStart = new HashMap<>(queued.get(2).segmentation); + Assert.assertEquals(longKey, viewStart.remove(ModuleViews.KEY_NAME)); + Assert.assertEquals("1", viewStart.remove(ModuleViews.KEY_VISIT)); + Assert.assertEquals("1", viewStart.remove(ModuleViews.KEY_START)); + Assert.assertNotNull(viewStart.remove(ModuleViews.KEY_SEGMENT)); + Assert.assertEquals(wide, viewStart); + Map viewEnd = new HashMap<>(queued.get(3).segmentation); + Assert.assertEquals(longKey, viewEnd.remove(ModuleViews.KEY_NAME)); + Assert.assertNotNull(viewEnd.remove(ModuleViews.KEY_SEGMENT)); + Assert.assertEquals(wide, viewEnd); + Assert.assertEquals(longValue, queued.get(4).segmentation.get("comment")); + + Map properties = new HashMap<>(wide); + properties.put(PredefinedUserPropertyKeys.NAME, longValue); + Countly.instance().userProfile().setProperties(properties); + Countly.instance().userProfile().push(longKey + "list", longValue); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = null; + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("user_details")) { + details = new JSONObject(request.get("user_details")); + } + } + Assert.assertNotNull(details); + Assert.assertEquals(longValue, details.getString(PredefinedUserPropertyKeys.NAME)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(151, custom.length()); + for (Map.Entry entry : wide.entrySet()) { + Assert.assertEquals(entry.getValue(), custom.getString(entry.getKey())); + } + Assert.assertEquals(longValue, custom.getJSONObject(longKey + "list").getString("$push")); + + Countly.instance().crashes().addCrashBreadcrumb(longValue); + Exception exception = new Exception(longValue); + Countly.instance().crashes().recordHandledException(exception, wide); + TestUtils.letTheClockCatchUp(); + DumpingCrashProcessor.replaceWithDump = true; + DumpingCrashProcessor.framesPerThread = 40; + Countly.instance().crashes().recordHandledException(new Exception("dumped")); + TestUtils.letTheClockCatchUp(); + + List crashes = crashRequests(); + Assert.assertEquals(2, crashes.size()); + Assert.assertEquals(stackTraceOf(exception), crashes.get(0).getString("_error")); + Assert.assertEquals(longValue, crashes.get(0).getString("_logs")); + JSONObject segments = crashes.get(0).getJSONObject("_custom"); + Assert.assertEquals(wide.size(), segments.length()); + for (Map.Entry entry : wide.entrySet()) { + Assert.assertEquals(entry.getValue(), segments.getString(entry.getKey())); + } + String dump = crashes.get(1).getString("_error"); + Assert.assertEquals(80, dump.split("\tat ly.count.Example.method", -1).length - 1); + Assert.assertTrue(dump.contains("Thread [worker-thread]:\n")); + Assert.assertEquals(3, crashes.get(1).getJSONObject("_custom").length()); + Assert.assertEquals("processor_breadcrumb", crashes.get(1).getString("_logs")); + + Assert.assertEquals(0, warningsContaining("limit of [")); + Assert.assertEquals(0, warningsContaining("reserved key")); + } + + /** + * The limits most scenarios run with: keys of 5 characters, values of 6, 2 segmentation entries + * and 3 stack trace lines per thread of 20 characters each, with networking off. + * + * @return the settings + */ + private static ServerConfigBuilder smallLimits() { + return new ServerConfigBuilder().keyLengthLimit(5).valueSizeLimit(6).segmentationValuesLimit(2) + .traceLinesLimit(3).traceLengthLimit(20).networking(false); + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings test configuration with a listener that keeps every warning. + * + * @return the configuration + */ + private Config configWithWarnings() { + return TestUtils.getConfigSdkBehaviorSettings().setLogListener(this::keepWarning); + } + + /** + * A log listener that keeps every warning. + * + * @param message the log line + * @param level its level + */ + private void keepWarning(String message, Config.LoggingLevel level) { + if (level == Config.LoggingLevel.WARN) { + warnings.add(message); + } + } + + /** + * How many warnings so far contain a text. + * + * @param text the text + * @return the number of warnings + */ + private int warningsContaining(String text) { + int count = 0; + for (String warning : warnings) { + if (warning.contains(text)) { + count++; + } + } + return count; + } + + /** + * Every event of every stored request, in queue order. + * + * @return the events + */ + private static List sentEvents() { + List events = new ArrayList<>(); + int requestCount = TestUtils.getCurrentRQ().length; + for (int i = 0; i < requestCount; i++) { + events.addAll(TestUtils.readEventsFromRequest(i, TestUtils.DEVICE_ID)); + } + return events; + } + + /** + * The keys of events, in order. + * + * @param events the events + * @return the keys + */ + private static List keysOf(List events) { + List keys = new ArrayList<>(); + for (EventImpl event : events) { + keys.add(event.key); + } + return keys; + } + + /** + * Asserts the key and the whole segmentation of an event. + * + * @param event the event + * @param key the expected key + * @param segmentation the expected segmentation + */ + private static void assertEvent(EventImpl event, String key, Map segmentation) { + Assert.assertEquals(key, event.key); + Assert.assertEquals(segmentation, new HashMap<>(event.segmentation)); + } + + /** + * The {@code user_details} object of a stored request. + * + * @param requestIndex the index of the request in the queue + * @return the user details + */ + private static JSONObject userDetails(int requestIndex) { + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertTrue("no request at index " + requestIndex + " in " + Arrays.toString(requests), requests.length > requestIndex); + return new JSONObject(requests[requestIndex].get("user_details")); + } + + /** + * The crash of every stored crash request, in queue order. + * + * @return the crashes + */ + private static List crashRequests() { + List crashes = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("crash")) { + crashes.add(new JSONObject(request.get("crash"))); + } + } + return crashes; + } + + /** + * The stack trace of a throwable as the SDK prints it. + * + * @param throwable the throwable + * @return the stack trace + */ + private static String stackTraceOf(Throwable throwable) { + StringWriter writer = new StringWriter(); + throwable.printStackTrace(new PrintWriter(writer)); + return writer.toString(); + } + + /** + * Cuts every line of a text to a length, keeping its line breaks. + * + * @param text the text + * @param maxLength the length + * @return the cut text + */ + private static String truncateLines(String text, int maxLength) { + StringBuilder truncated = new StringBuilder(); + String[] lines = text.split("\n", -1); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + String carriageReturn = ""; + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + carriageReturn = "\r"; + } + truncated.append(line.length() > maxLength ? line.substring(0, maxLength) : line).append(carriageReturn); + if (i < lines.length - 1) { + truncated.append('\n'); + } + } + return truncated.toString(); + } + + /** + * The number of line breaks of a text. + * + * @param text the text + * @return the number of lines that end with a line break + */ + private static int lineCount(String text) { + return text.split("\n", -1).length - 1; + } + + /** + * A text repeated. + * + * @param text the text + * @param times how many times + * @return the repeated text + */ + private static String repeat(String text, int times) { + StringBuilder repeated = new StringBuilder(text.length() * times); + for (int i = 0; i < times; i++) { + repeated.append(text); + } + return repeated.toString(); + } + + /** + * A crash processor that, when asked to, replaces the stack trace with a dump of a main and a + * worker thread and sets its own segments and breadcrumbs, as a customer's processor may. + */ + public static class DumpingCrashProcessor implements CrashProcessor { + static volatile boolean replaceWithDump = false; + static volatile int framesPerThread = 5; + private static final Thread MAIN_THREAD = new Thread(() -> { + }, "main"); + private static final Thread WORKER_THREAD = new Thread(() -> { + }, "worker-thread"); + + /** + * Replaces the stack trace, the segments and the breadcrumbs when asked to. + * + * @param crash the crash + * @return the same crash, never vetoed + */ + @Override + public Crash process(Crash crash) { + if (replaceWithDump) { + Map traces = new LinkedHashMap<>(); + traces.put(MAIN_THREAD, frames(framesPerThread)); + traces.put(WORKER_THREAD, frames(framesPerThread)); + crash.addTraces(MAIN_THREAD, traces); + Map segments = new HashMap<>(); + segments.put("processor_segment", "processor_value"); + segments.put("another_segment", "another_value"); + segments.put("zzz", "z"); + crash.setSegments(segments); + crash.setLogs(new String[] { "processor_breadcrumb" }); + } + return crash; + } + + /** + * Stack trace lines of a made up class. + * + * @param count how many + * @return the lines + */ + private static StackTraceElement[] frames(int count) { + StackTraceElement[] frames = new StackTraceElement[count]; + for (int i = 0; i < count; i++) { + frames[i] = new StackTraceElement("ly.count.Example", "method" + i, "Example.java", i + 1); + } + return frames; + } + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationNetworkTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationNetworkTests.java new file mode 100644 index 00000000..8c85c1eb --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationNetworkTests.java @@ -0,0 +1,1018 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.BooleanSupplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The network resilience settings of the SDK behavior settings, observed through the public API: the + * backoff mechanism, {@code bom}, {@code bom_at}, {@code bom_rqp}, {@code bom_ra} and {@code bom_d}, + * and the request drop age, {@code dort}. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread. Queued requests go through the real networking stack to a local HTTP server + * that can hold each answer back and records when every request arrived and when its answer started, + * so the gap a backoff leaves between two requests can be measured. Backoff settings are set to + * seconds, never left at their defaults, and no wait relies on the 60 second SDK timer. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationNetworkTests { + + private static final long SLOW_ANSWER_MS = 1_200; + //longer than the wait budget, so a backoff that should not happen fails the wait instead of slowing it + private static final int BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS = 60; + private static final long WAIT_BUDGET_MS = 30_000; + private static final long POLL_MS = 50; + private static final long TIMER_SLACK_MS = 50; + private static final long LATE_RESUME_MS = 5_000; + private static final long MINUTE_MS = 60L * 1000L; + private static final long HOUR_MS = 60L * MINUTE_MS; + + private static final String BACKOFF_STARTED = "holding the request queue back for ["; + private static final String BACKOFF_ENDED = "the backoff is over"; + private static final String NOT_BACKING_OFF = "not backing off"; + private static final String DROPPED_UNSENT = "dropping it without sending"; + private static final String CONTENT_ENDPOINT = "/o/sdk/content?"; + private static final String NO_CONTENT_RESPONSE = "{\"jsonArray\":[{\"result\":\"No content block found!\"}]}"; + + private SettingsAndContentServer server; + private QueueServer queueServer; + private final List logs = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module and with fresh fakes. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new SettingsAndContentServer(); + queueServer = null; + logs.clear(); + } + + /** + * Stops the SDK, clearing its data, and the local HTTP server when one was started. + */ + @After + public void afterTest() { + Countly.instance().halt(); + if (queueServer != null) { + queueServer.stop(); + } + } + + // region scenarios + + /** + * A request the server answers slowly while few requests are queued holds the queue back for + * {@code bom_d}, then the queue drains on its own. + *

+ * Verifies against a local server that after an answer slower than {@code bom_at}, with two + * requests left, the next request arrives only once {@code bom_d} has passed since the server + * started answering; that meanwhile a newly recorded request and nudges of the send loop send + * nothing and the queue stays on disk as it was; that the backoff ends without any nudge; and + * that the fast answers after it hold nothing back. + */ + @Test + public void backoff_slowAnswerWithFewRequestsQueued_holdsTheQueueBackForTheDuration_thenItDrainsOnItsOwn() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffAcceptedTimeout(1).backoffDuration(3)); + init(networkedConfig()); + + recordEvents("first", "second", "third"); + awaitLogs(BACKOFF_STARTED, 1); + Assert.assertEquals(1, logsContaining("and [2] requests remain queued, " + BACKOFF_STARTED + "3] seconds")); + Assert.assertEquals(1, queueServer.received.size()); + + Countly.instance().events().recordEvent("during_backoff"); + TestUtils.letTheClockCatchUp(); + Map[] held = TestUtils.getCurrentRQ(); + Assert.assertEquals(3, held.length); + Assert.assertEquals("[\"second\"]", eventKeys(held[0]).toString()); + Assert.assertEquals("[\"third\"]", eventKeys(held[1]).toString()); + Assert.assertEquals("[\"during_backoff\"]", eventKeys(held[2]).toString()); + + nudgeUntil(queueServer.received.get(0).answeredAtNs + 2_000L * 1_000_000L); + Assert.assertEquals(1, queueServer.received.size()); + Assert.assertEquals(3, queuedRequestCount()); + Assert.assertEquals(0, logsContaining(BACKOFF_ENDED)); + + Assert.assertTrue("the queue never drained after the backoff, logs: " + logs, waitFor(() -> queueServer.received.size() == 4 && queuedRequestCount() == 0)); + long gapMs = gapBeforeRequestMs(1); + Assert.assertTrue("the next request came [" + gapMs + "] ms after the slow answer", gapMs >= 3_000 - TIMER_SLACK_MS); + Assert.assertTrue("the queue resumed [" + gapMs + "] ms after the slow answer", gapMs < 3_000 + LATE_RESUME_MS); + Assert.assertEquals(Arrays.asList("first", "second", "third", "during_backoff"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(1, logsContaining(BACKOFF_ENDED)); + Assert.assertEquals(0, logsContaining(NOT_BACKING_OFF)); + } + + /** + * The backoff needs a slow answer and the requests still queued within {@code bom_rqp} of the + * request queue size. + *

+ * Verifies against a local server, with {@code rqs} 4 and {@code bom_rqp} 0.25, so at most one + * request left, that a slow answer with two requests left holds nothing back, that the next slow + * answer, with one request left, holds the queue back for {@code bom_d}, and that the fast answer + * to the last request holds nothing back. + */ + @Test + public void backoff_needsTheQueueWithinItsShareOfTheQueueSize_andASlowAnswer() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().requestQueueSize(4).backoffRequestQueuePercentage(0.25) + .backoffAcceptedTimeout(1).backoffDuration(2)); + init(networkedConfig()); + + recordEvents("a", "b", "c"); + Assert.assertTrue("the queue never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 3 && queuedRequestCount() == 0)); + + Assert.assertEquals(Arrays.asList("a", "b", "c"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining("but [2] requests remain queued, more than [1.0], " + NOT_BACKING_OFF)); + Assert.assertEquals(1, logsContaining("and [1] requests remain queued, " + BACKOFF_STARTED + "2] seconds")); + Assert.assertEquals(1, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(1, logsContaining(NOT_BACKING_OFF)); + long gapAfterFirstMs = gapBeforeRequestMs(1); + long gapAfterSecondMs = gapBeforeRequestMs(2); + Assert.assertTrue("the second request waited [" + gapAfterFirstMs + "] ms", gapAfterFirstMs < 2_000); + Assert.assertTrue("the third request waited only [" + gapAfterSecondMs + "] ms", gapAfterSecondMs >= 2_000 - TIMER_SLACK_MS); + } + + /** + * The backoff never holds the queue back after a request older than {@code bom_ra}, nor while the + * server turns it off with {@code bom}. + *

+ * Verifies against a local server, with a {@code bom_d} longer than the wait budget so that a + * backoff fails the wait, that two stored requests recorded two hours ago are both sent although + * each is answered slowly, each slow answer logged as coming too late to back off; and, after a + * restart with {@code bom} false from the server, that two new requests answered slowly are sent + * one after the other without any backoff. + */ + @Test + public void backoff_neverAfterARequestOlderThanTheRequestAge_norWhileTheServerTurnsItOff() throws Exception { + startQueueServer(); + long twoHoursAgo = System.currentTimeMillis() - 2 * HOUR_MS; + storeRequest(twoHoursAgo, "old_1", twoHoursAgo); + storeRequest(twoHoursAgo + 1, "old_2", twoHoursAgo + 1); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffAcceptedTimeout(1).backoffRequestAge(1) + .backoffDuration(BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS)); + init(networkedConfig()); + + Assert.assertTrue("the old requests never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("old_1", "old_2"), receivedParams("marker")); + Assert.assertEquals(2, logsContaining("but the request is older than [1] hours, " + NOT_BACKING_OFF)); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + Countly.instance().halt(); + + queueServer.received.clear(); + logs.clear(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffMechanism(false).backoffAcceptedTimeout(1) + .backoffDuration(BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS)); + init(networkedConfig()); + Assert.assertFalse(provider().getBOMEnabled()); + + recordEvents("x", "y"); + Assert.assertTrue("the new requests never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("x", "y"), receivedEventKeys()); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(0, logsContaining(NOT_BACKING_OFF)); + } + + /** + * {@link Config#disableBackoffMechanism()} keeps the backoff off while the settings server stays + * silent, and a {@code bom} true from the server turns it back on. + *

+ * Verifies against a local server, with the settings server never answering and the provided + * settings setting {@code bom_at} and a {@code bom_d} longer than the wait budget, that slow + * answers hold nothing back; then that once a response sets {@code bom} true with a short + * {@code bom_d}, the next slow answer holds the queue back for that long. + */ + @Test + public void backoff_offByTheDeveloper_staysOffWithASilentServer_andTheServerTurnsItBackOn() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.pending(); + init(networkedConfig().disableBackoffMechanism().setSdkBehaviorSettings(new ServerConfigBuilder() + .backoffAcceptedTimeout(1).backoffDuration(BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS).build())); + Assert.assertFalse(provider().getBOMEnabled()); + Assert.assertEquals(1, provider().getBOMAcceptedTimeoutSeconds()); + + recordEvents("a", "b"); + Assert.assertTrue("the queue never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + + push(new ServerConfigBuilder().backoffMechanism(true).backoffDuration(2)); + Assert.assertTrue(provider().getBOMEnabled()); + Assert.assertEquals(2, provider().getBOMDuration()); + + recordEvents("c", "d"); + Assert.assertTrue("the queue never drained after the backoff, logs: " + logs, waitFor(() -> queueServer.received.size() == 4 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("a", "b", "c", "d"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining("and [1] requests remain queued, " + BACKOFF_STARTED + "2] seconds")); + long gapMs = gapBeforeRequestMs(3); + Assert.assertTrue("the last request waited only [" + gapMs + "] ms", gapMs >= 2_000 - TIMER_SLACK_MS); + } + + /** + * Backend mode never holds its request queue back and never drops a request for its age, whatever + * the resolved values say. + *

+ * Verifies against a local server, with the settings inert in backend mode and the resolved + * backoff and drop age values forced to ones that would back off and drop, that three requests, + * the first two answered slowly and the first recorded three hours ago, are all sent in order with + * their own timestamps, that nothing backs off or is dropped, and that no settings request is made. + */ + @Test + public void backendMode_neverHoldsTheQueueBack_norDropsARequestForItsAge() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + init(networkedConfig().enableBackendMode()); + ModuleConfiguration module = SDKCore.instance.module(ModuleConfiguration.class); + module.currentVBOMAcceptedTimeoutSeconds = 1; + module.currentVBOMDuration = BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS; + module.currentVDropAgeHours = 1; + Assert.assertTrue(provider().getBOMEnabled()); + + long threeHoursAgo = System.currentTimeMillis() - 3 * HOUR_MS; + Countly.instance().backendM().sessionBegin("device_old", null, null, threeHoursAgo); + Countly.instance().backendM().sessionBegin("device_fresh", null, null, null); + Countly.instance().backendM().sessionBegin("device_last", null, null, null); + + Assert.assertTrue("the memory queue never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 3 && memoryQueueSize() == 0)); + Assert.assertEquals(Arrays.asList("device_old", "device_fresh", "device_last"), receivedParams("device_id")); + Assert.assertEquals(String.valueOf(threeHoursAgo), queueServer.received.get(0).params.get("timestamp")); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(0, logsContaining(NOT_BACKING_OFF)); + Assert.assertEquals(0, logsContaining(DROPPED_UNSENT)); + Assert.assertTrue(server.settings.requests.isEmpty()); + } + + /** + * Stopping the SDK while the queue is held back ends the backoff for good. + *

+ * Verifies against a local server that once the SDK stopped during a backoff, nothing is sent + * after the backoff would have ended, the held request stays on disk, the backoff thread is gone + * and the backoff never reports its end, and that the next init sends the held request. + */ + @Test + public void backoff_stopWhileHeldBack_sendsNothingAfterwards_leavesNoThread_andTheNextInitSendsTheHeldRequest() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffAcceptedTimeout(1).backoffDuration(2)); + Set threadsBefore = backoffThreads(); + init(networkedConfig()); + + recordEvents("first", "held"); + awaitLogs(BACKOFF_STARTED, 1); + Set threads = backoffThreads(); + threads.removeAll(threadsBefore); + Assert.assertEquals(1, threads.size()); + Assert.assertEquals(1, queuedRequestCount()); + + Countly.instance().stop(); + Assert.assertTrue("the backoff thread outlived stop()", waitFor(() -> noneAlive(threads))); + sleepUntil(queueServer.received.get(0).answeredAtNs + 3_500L * 1_000_000L); + Assert.assertEquals(1, queueServer.received.size()); + Assert.assertEquals(1, queuedRequestCount()); + Assert.assertEquals(0, logsContaining(BACKOFF_ENDED)); + + init(networkedConfig()); + Assert.assertTrue("the held request was never sent, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("first", "held"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining(BACKOFF_STARTED)); + } + + /** + * {@code dort} drops the stored requests older than it without sending them and sends the others, + * and {@code dort} 0 drops nothing. + *

+ * Verifies against a local server that the age of a request comes from its {@code timestamp} + * parameter, so an old storage ID with a fresh timestamp is sent and a fresh storage ID with an + * old timestamp is dropped, and from its storage ID when it has no timestamp; that each drop is + * logged with the request it dropped; and that with {@code dort} 0 the same old requests are all + * sent in order. + */ + @Test + public void requestDropAge_dropsTheStoredRequestsOlderThanIt_unsent_andZeroDropsNothing() throws Exception { + startQueueServer(); + long now = System.currentTimeMillis(); + long threeHoursAgo = now - 3 * HOUR_MS; + storeRequest(threeHoursAgo, "old_timestamp", threeHoursAgo); + storeRequest(threeHoursAgo + 1, "fresh_timestamp_old_id", now); + storeRequest(threeHoursAgo + 2, "no_timestamp_old_id", null); + storeRequest(now - 2 * MINUTE_MS, "old_timestamp_fresh_id", now - 2 * HOUR_MS); + storeRequest(now - MINUTE_MS, "within_the_drop_age", now - 30 * MINUTE_MS); + server.settings.failing(); + init(networkedConfig().setSdkBehaviorSettings(new ServerConfigBuilder().dropOldRequestTime(1).build())); + Assert.assertEquals(1, provider().getRequestDropAgeHours()); + + Assert.assertTrue("the queue never drained, logs: " + logs, waitNudging(() -> queuedRequestCount() == 0 && queueServer.received.size() == 2)); + Assert.assertEquals(Arrays.asList("fresh_timestamp_old_id", "within_the_drop_age"), receivedParams("marker")); + Assert.assertEquals(3, logsContaining(DROPPED_UNSENT)); + for (long droppedId : new long[] { threeHoursAgo, threeHoursAgo + 2, now - 2 * MINUTE_MS }) { + Assert.assertEquals(1, logsContaining("request [" + droppedId + "] is older than the request drop age of [1] hours")); + } + + TestUtils.createCleanTestState(); + queueServer.received.clear(); + logs.clear(); + storeRequest(threeHoursAgo, "old_timestamp", threeHoursAgo); + storeRequest(threeHoursAgo + 2, "no_timestamp_old_id", null); + storeRequest(now - 2 * MINUTE_MS, "old_timestamp_fresh_id", now - 2 * HOUR_MS); + init(networkedConfig().setSdkBehaviorSettings(new ServerConfigBuilder().dropOldRequestTime(0).build())); + Assert.assertEquals(0, provider().getRequestDropAgeHours()); + + Assert.assertTrue("the queue never drained, logs: " + logs, waitNudging(() -> queuedRequestCount() == 0 && queueServer.received.size() == 3)); + Assert.assertEquals(Arrays.asList("old_timestamp", "no_timestamp_old_id", "old_timestamp_fresh_id"), receivedParams("marker")); + Assert.assertEquals(0, logsContaining(DROPPED_UNSENT)); + } + + /** + * A journey trigger whose request is dropped for its age is settled, while a later trigger still + * refreshes the content zone. + *

+ * Verifies against a local server that a trigger request recorded while networking is off and + * aged past {@code dort} on disk is dropped unsent once networking is back on, that its owner is + * told with {@link Transport#NO_RESPONSE_CODE}, which leaves no pending trigger and the zone + * inactive, and that a fresh trigger is then sent, accepted and enters the zone. + */ + @Test + public void requestDropAge_settlesTheJourneyTriggerOfADroppedRequest_andALaterTriggerStillRefreshes() throws Exception { + startQueueServer(); + server.settings.failing(); + init(networkedConfig().enableFeatures(Config.Feature.Content).setEventQueueSizeToSend(100) + .setSdkBehaviorSettings(new ServerConfigBuilder().networking(false).journeyTriggerEvents(names("purchase")).dropOldRequestTime(1).build())); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(new FakeDisplay()); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("purchase"); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + Set pending = SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds; + Assert.assertEquals(storedRequestIds(), pending); + Assert.assertEquals(1, pending.size()); + ageStoredRequests(2 * HOUR_MS); + + push(new ServerConfigBuilder().networking(true)); + Assert.assertTrue("the aged request was never dropped, logs: " + logs, waitNudging(() -> queuedRequestCount() == 0)); + Assert.assertEquals(1, logsContaining(DROPPED_UNSENT)); + Assert.assertEquals(1, logsContaining("carrying a journey trigger failed with code [" + Transport.NO_RESPONSE_CODE + "], the content zone is not refreshed")); + Assert.assertTrue(pending.isEmpty()); + Assert.assertTrue(queueServer.received.isEmpty()); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("purchase"); + awaitZoneFetch(1); + Assert.assertEquals(Collections.singletonList("purchase"), receivedEventKeys()); + Assert.assertTrue(pending.isEmpty()); + Assert.assertEquals(1, logsContaining(DROPPED_UNSENT)); + } + + /** + * The request queue trim settles the journey triggers of the requests it drops unread. + *

+ * Verifies on disk, with networking off and {@code rqs} 2, that each journey trigger waits for + * its own request, and that a third trigger, whose request drops the oldest one, leaves waiting + * only the triggers of the two requests still queued. + */ + @Test + public void requestQueueTrim_settlesTheJourneyTriggersOfTheRequestsItDrops() { + server.settings.respondWith(new ServerConfigBuilder().networking(false).requestQueueSize(2).journeyTriggerEvents(names("purchase"))); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(100) + .setLogListener((message, level) -> logs.add(message))); + Set pending = SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds; + + recordEvents("purchase", "purchase"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Set firstTwo = storedRequestIds(); + Assert.assertEquals(firstTwo, pending); + long oldest = Collections.min(firstTwo); + + recordEvents("purchase"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Set lastTwo = storedRequestIds(); + Assert.assertFalse(lastTwo.contains(oldest)); + Assert.assertEquals(lastTwo, pending); + Assert.assertEquals(1, logsContaining("the request [" + oldest + "] carrying a journey trigger was dropped from the queue without being sent")); + } + + // endregion + // region helpers + + /** + * Stands in for the settings and the content endpoints of the server. Settings requests go to a + * {@link ModuleConfigurationTests.ServerConfigResponder}; content fetches are recorded and + * answered on the calling thread with no content, unless networking is off. + */ + private static final class SettingsAndContentServer implements ImmediateRequestGenerator { + final ModuleConfigurationTests.ServerConfigResponder settings = new ModuleConfigurationTests.ServerConfigResponder(); + final List contentFetches = new CopyOnWriteArrayList<>(); + + /** + * A request maker that routes by endpoint. + * + * @return the request maker + */ + @Override + public ImmediateRequestI createImmediateRequestMaker() { + ImmediateRequestI settingsRequestMaker = settings.createImmediateRequestMaker(); + return (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> { + if (!CONTENT_ENDPOINT.equals(customEndpoint)) { + settingsRequestMaker.doWork(requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log); + return; + } + if (!networkingIsEnabled) { + callback.callback(null); + return; + } + contentFetches.add(requestData); + callback.callback(new JSONObject(NO_CONTENT_RESPONSE)); + }; + } + } + + /** + * One request as the local server saw it. + */ + private static final class Received { + final Map params; + final long receivedAtNs; + volatile long answeredAtNs; + + /** + * Records a request as it arrives. + * + * @param params the decoded request parameters + * @param receivedAtNs {@link System#nanoTime()} when it arrived + */ + Received(Map params, long receivedAtNs) { + this.params = params; + this.receivedAtNs = receivedAtNs; + } + } + + /** + * A local HTTP server standing in for the request queue endpoint: it accepts every request, holds + * back the answers a test asks it to, and records each request with the moment it arrived and the + * moment its answer started. One handler thread, so requests are recorded in the order they came. + */ + private static final class QueueServer { + final HttpServer http; + final List received = new CopyOnWriteArrayList<>(); + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final Queue answerDelaysMs = new ConcurrentLinkedQueue<>(); + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + QueueServer() throws IOException { + http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + http.createContext("/", this::handle); + http.setExecutor(executor); + http.start(); + } + + /** + * Holds back the answers to the next requests, one delay per request in arrival order. Any + * request after those is answered right away. + * + * @param delaysMs how long to hold each answer back, in milliseconds + */ + void holdTheNextAnswers(long... delaysMs) { + for (long delayMs : delaysMs) { + answerDelaysMs.add(delayMs); + } + } + + /** + * Records a request, holds its answer back when asked to, then accepts it. + * + * @param exchange the request and its response + * @throws IOException when the response cannot be written + */ + private void handle(HttpExchange exchange) throws IOException { + String query = exchange.getRequestURI().getRawQuery(); + String body = new String(readAll(exchange.getRequestBody()), StandardCharsets.UTF_8); + Received request = new Received(parseParams(query != null && !query.isEmpty() ? query : body), System.nanoTime()); + received.add(request); + + Long delayMs = answerDelaysMs.poll(); + if (delayMs != null && delayMs > 0) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + byte[] response = "{\"result\":\"Success\"}".getBytes(StandardCharsets.UTF_8); + request.answeredAtNs = System.nanoTime(); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + + /** + * The port the server listens on. + * + * @return the port + */ + int port() { + return http.getAddress().getPort(); + } + + /** + * Stops the server and its handler thread, even while it holds an answer back. + */ + void stop() { + http.stop(0); + executor.shutdownNow(); + } + } + + /** + * A display that shows nothing, so the content zone can be entered. + */ + private static final class FakeDisplay implements ContentDisplay { + /** + * A landscape screen. + * + * @return the screen + */ + @Override + public ContentScreen getScreen() { + return new ContentScreen(1600, 900); + } + + /** + * Shows nothing. + * + * @param content the content to show + * @param onClosed called when the content closes + */ + @Override + public void present(ContentData content, ContentCloseCallback onClosed) { + } + } + + /** + * Starts {@link #queueServer}. + * + * @throws IOException when the server cannot start + */ + private void startQueueServer() throws IOException { + queueServer = new QueueServer(); + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests and content fetches. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.settings.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * A configuration pointing at {@link #queueServer}, with events sent one per request, no request + * cooldowns, and every log line kept. + * + * @return the configuration + */ + private Config networkedConfig() { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + return new Config("http://127.0.0.1:" + queueServer.port(), TestUtils.SERVER_APP_KEY, directory) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setNetworkRequestCooldown(0) + .setNetworkImportantRequestCooldown(0) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * Records events one after another, letting the clock move on after each, so every request gets + * a storage ID of its own. + * + * @param keys the event keys + */ + private static void recordEvents(String... keys) { + for (String key : keys) { + Countly.instance().events().recordEvent(key); + TestUtils.letTheClockCatchUp(); + } + } + + /** + * Stores a request file before init, as an earlier run would have left it. + * + * @param id the storage ID, which names the file + * @param marker a parameter that tells the requests apart on the server + * @param timestamp the {@code timestamp} parameter, {@code null} for none + * @throws IOException when the file cannot be written + */ + private static void storeRequest(long id, String marker, Long timestamp) throws IOException { + Request request = new Request(id); + request.params = new Params("app_key", TestUtils.SERVER_APP_KEY, "device_id", TestUtils.DEVICE_ID, "marker", marker); + if (timestamp != null) { + request.params.add("timestamp", timestamp); + } + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + Files.write(new File(directory, "[CLY]_request_" + id).toPath(), request.store(new Log(Config.LoggingLevel.OFF, null))); + } + + /** + * Moves the {@code timestamp} parameter of every stored request back by the given time, as if it + * had been recorded that much earlier, keeping its storage ID. + * + * @param byMs how far back, in milliseconds + * @throws IOException when a file cannot be rewritten + */ + private static void ageStoredRequests(long byMs) throws IOException { + Pattern timestamp = Pattern.compile("(^|&)timestamp=(\\d+)"); + for (File file : requestFiles()) { + String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + Matcher matcher = timestamp.matcher(content); + Assert.assertTrue("no timestamp in " + file.getName(), matcher.find()); + long aged = Long.parseLong(matcher.group(2)) - byMs; + String agedContent = content.substring(0, matcher.start(2)) + aged + content.substring(matcher.end(2)); + Files.write(file.toPath(), agedContent.getBytes(StandardCharsets.UTF_8)); + } + } + + /** + * The stored request files. + * + * @return the files, never {@code null} + */ + private static File[] requestFiles() { + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith("[CLY]_request_")); + return files == null ? new File[0] : files; + } + + /** + * Counts the stored requests without reading them, as the network loop can remove one while it + * is read. + * + * @return the number of stored requests + */ + private static int queuedRequestCount() { + return requestFiles().length; + } + + /** + * The storage IDs of the stored requests. + * + * @return the IDs + */ + private static Set storedRequestIds() { + Set ids = new HashSet<>(); + for (File file : requestFiles()) { + ids.add(Long.parseLong(file.getName().substring("[CLY]_request_".length()))); + } + return ids; + } + + /** + * How many requests the memory queue of backend mode holds. + * + * @return the number of requests + */ + private static int memoryQueueSize() { + SDKCore core = SDKCore.instance; + synchronized (core.lockBRQStorage) { + return core.requestQueueMemory.size(); + } + } + + /** + * The key of the event each request the server received carried, in the order they arrived. + * + * @return the event keys + */ + private List receivedEventKeys() { + List keys = new ArrayList<>(); + for (Received request : queueServer.received) { + JSONArray requestKeys = eventKeys(request.params); + for (int i = 0; i < requestKeys.length(); i++) { + keys.add(requestKeys.getString(i)); + } + } + return keys; + } + + /** + * One parameter of each request the server received, in the order they arrived. + * + * @param key the parameter + * @return its values + */ + private List receivedParams(String key) { + List values = new ArrayList<>(); + for (Received request : queueServer.received) { + values.add(request.params.get(key)); + } + return values; + } + + /** + * How long the server had to wait for a request after it started answering the one before it. + * + * @param index the position of the request, from 1 + * @return the gap in milliseconds + */ + private long gapBeforeRequestMs(int index) { + Received previous = queueServer.received.get(index - 1); + Received next = queueServer.received.get(index); + return (next.receivedAtNs - previous.answeredAtNs) / 1_000_000L; + } + + /** + * The keys of the events a request carries, in order. + * + * @param params the parameters of the request + * @return the keys as a JSON array, empty when the request carries no events + */ + private static JSONArray eventKeys(Map params) { + JSONArray keys = new JSONArray(); + String events = params.get("events"); + if (events == null) { + return keys; + } + JSONArray eventArray = new JSONArray(events); + for (int i = 0; i < eventArray.length(); i++) { + keys.put(eventArray.getJSONObject(i).getString("key")); + } + return keys; + } + + /** + * The live backoff threads. + * + * @return the threads + */ + private static Set backoffThreads() { + Set threads = new HashSet<>(); + for (Thread thread : Thread.getAllStackTraces().keySet()) { + if (DefaultNetworking.BACKOFF_THREAD_NAME.equals(thread.getName()) && thread.isAlive()) { + threads.add(thread); + } + } + return threads; + } + + /** + * Whether every given thread has ended. + * + * @param threads the threads + * @return {@code true} when none is alive + */ + private static boolean noneAlive(Set threads) { + for (Thread thread : threads) { + if (thread.isAlive()) { + return false; + } + } + return true; + } + + /** + * Holds the first tick of every zone timer an hour away, so the zone fetches only when a test + * ticks it by hand. + */ + private static void holdTheZoneTimer() { + contentModule().startedAtForTests(System.currentTimeMillis() + HOUR_MS); + } + + /** + * The content module of the running SDK. + * + * @return the module + */ + private static ModuleContent contentModule() { + return SDKCore.instance.module(ModuleContent.class); + } + + /** + * Ticks the zone once by hand and asserts that it fetched nothing, as no zone is active. + * + * @param fetchesSoFar the content fetches made so far + */ + private void assertZoneIsInactive(int fetchesSoFar) { + contentModule().onZoneTimerTick(); + Assert.assertEquals(fetchesSoFar, server.contentFetches.size()); + } + + /** + * Ticks the zone by hand until it fetches, which it does once a refresh entered it. + * + * @param expectedFetches the content fetches expected once it fetched + */ + private void awaitZoneFetch(int expectedFetches) throws InterruptedException { + boolean fetched = waitNudging(() -> { + contentModule().onZoneTimerTick(); + return server.contentFetches.size() >= expectedFetches; + }); + Assert.assertTrue("the content zone never fetched, logs: " + logs, fetched); + Assert.assertEquals(expectedFetches, server.contentFetches.size()); + } + + /** + * Waits until the log holds a number of lines containing a text. + * + * @param text the text + * @param count how many lines + */ + private void awaitLogs(String text, int count) throws InterruptedException { + Assert.assertTrue("never logged [" + text + "] " + count + " times, logs: " + logs, waitFor(() -> logsContaining(text) >= count)); + Assert.assertEquals(count, logsContaining(text)); + } + + /** + * How many log lines so far contain a text. + * + * @param text the text + * @return the number of lines + */ + private int logsContaining(String text) { + int count = 0; + for (String line : logs) { + if (line.contains(text)) { + count++; + } + } + return count; + } + + /** + * Makes the send loop look at the queue, as a newly stored request does. + */ + private static void nudge() { + SDKCore core = SDKCore.instance; + if (core != null && core.networking != null && core.config != null) { + core.networking.check(core.config); + } + } + + /** + * Nudges the send loop every {@link #POLL_MS} until the given moment. + * + * @param deadlineNs the moment, as {@link System#nanoTime()} + */ + private static void nudgeUntil(long deadlineNs) throws InterruptedException { + while (System.nanoTime() < deadlineNs) { + nudge(); + Thread.sleep(POLL_MS); + } + } + + /** + * Sleeps until the given moment. + * + * @param deadlineNs the moment, as {@link System#nanoTime()} + */ + private static void sleepUntil(long deadlineNs) throws InterruptedException { + long remainingMs = (deadlineNs - System.nanoTime()) / 1_000_000L; + if (remainingMs > 0) { + Thread.sleep(remainingMs); + } + } + + /** + * Polls until the condition holds or {@link #WAIT_BUDGET_MS} runs out, without nudging the send + * loop, so the SDK has to drain its queue on its own. + * + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitFor(BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_BUDGET_MS; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(POLL_MS); + } + return condition.getAsBoolean(); + } + + /** + * Polls like {@link #waitFor(BooleanSupplier)}, nudging the send loop on every round. + * + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitNudging(BooleanSupplier condition) throws InterruptedException { + return waitFor(() -> { + nudge(); + return condition.getAsBoolean(); + }); + } + + /** + * Decodes url encoded request parameters. + * + * @param data the parameters + * @return the parameters by name + */ + private static Map parseParams(String data) { + Map params = new HashMap<>(); + for (String pair : data.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return params; + } + + /** + * Reads a stream to its end. + * + * @param stream the stream + * @return the bytes + * @throws IOException when the stream cannot be read + */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java new file mode 100644 index 00000000..0cfa1c4e --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java @@ -0,0 +1,1145 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpServer; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.math.BigDecimal; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * SDK behavior settings, the server side configuration served by {@code /o/sdk?method=sc}: how + * {@link ModuleConfiguration} validates, merges, stores and resolves them, and how the SDK applies + * the values it owns at init and when a response changes them. + *

+ * The settings request goes through a {@link ServerConfigResponder} injected on the + * {@link InternalConfig} before init. It answers on the calling thread, so the response of the fetch + * made at init is applied before init returns, and a later response is pushed by calling + * {@link ModuleConfiguration#fetchConfigFromServer(InternalConfig)} the way the refresh timer does. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationTests { + + private static final Set TOP_LEVEL_KEYS = names("v", "t", "c", "lg", "ct"); + + private ServerConfigResponder server; + + /** + * Starts every test from empty storage, without a test module and with a fresh responder. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ServerConfigResponder(); + } + + /** + * Stops the SDK, clearing its data, and removes the test module. + */ + @After + public void afterTest() { + Countly.instance().halt(); + SDKCore.testDummyModule = null; + } + + // region scenarios + + /** + * The type table of every settings key, run through the validation a response goes through. + *

+ * Verifies that each of the 42 keys keeps every valid value and loses every value of the wrong + * type or range, that unknown keys are dropped, and that a decimal, which the JSON parser hands + * over as a {@link BigDecimal}, is still accepted as {@code bom_rqp}. + */ + @Test + public void removeUnsupportedKeys_keepsEveryValidValueAndDropsTheRest() throws IllegalAccessException { + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + ModuleConfiguration module = module(); + + Map> valid = new LinkedHashMap<>(); + Map> invalid = new LinkedHashMap<>(); + for (String key : Arrays.asList("tracking", "networking", "st", "vt", "cet", "ecz", "crt", "ast", "avt", "acr", "lt", "rcz", "bom", "log", "cr")) { + valid.put(key, Arrays.asList("true", "false")); + invalid.put(key, Arrays.asList("1", "0", "\"true\"", "null", "{}", "[]")); + } + for (String key : Arrays.asList("scui", "rqs", "eqs", "sui", "lkl", "lvs", "lsv", "lbc", "ltlpt", "ltl", "upcl", "bom_at", "bom_ra", "bom_d")) { + valid.put(key, Arrays.asList("1", "7", "2147483647")); + invalid.put(key, Arrays.asList("0", "-5", "1.5", "5.0", "\"5\"", "true", "3000000000", "null")); + } + valid.put("dort", Arrays.asList("0", "1", "72")); + invalid.put("dort", Arrays.asList("-1", "0.5", "\"1\"", "false")); + valid.put("czi", Arrays.asList("16", "300")); + invalid.put("czi", Arrays.asList("15", "0", "-16", "16.0", "\"16\"")); + valid.put("bom_rqp", Arrays.asList("0.5", "0.01", "0.99", "1e-1")); + invalid.put("bom_rqp", Arrays.asList("0", "1", "0.0", "1.0", "-0.5", "1.5", "\"0.5\"", "true", "null")); + for (String key : Arrays.asList("eb", "ew", "sb", "sw", "upb", "upw", "jte", "jtv")) { + valid.put(key, Arrays.asList("[]", "[\"a\",\"b\"]")); + invalid.put(key, Arrays.asList("\"a\"", "{}", "1", "true", "null")); + } + for (String key : Arrays.asList("esb", "esw")) { + valid.put(key, Arrays.asList("{}", "{\"purchase\":[\"card\"]}")); + invalid.put(key, Arrays.asList("[]", "\"a\"", "1", "null")); + } + + Set settingsKeys = new HashSet<>(settingsKeysOfTheModule()); + Assert.assertEquals("the table covers every settings key the module declares", settingsKeys, valid.keySet()); + Assert.assertEquals(42, settingsKeys.size()); + + for (Map.Entry> entry : valid.entrySet()) { + for (String literal : entry.getValue()) { + assertKeptByValidation(module, entry.getKey(), literal, true); + } + } + for (Map.Entry> entry : invalid.entrySet()) { + for (String literal : entry.getValue()) { + assertKeptByValidation(module, entry.getKey(), literal, false); + } + } + for (String unknownKey : Arrays.asList("heartbeat", "TRACKING", "someFutureFeature", "")) { + assertKeptByValidation(module, unknownKey, "true", false); + } + + JSONObject settings = validated(module, "{\"bom_rqp\":0.25,\"tracking\":\"no\",\"eqs\":3}"); + Assert.assertTrue(settings.get("bom_rqp") instanceof BigDecimal); + Assert.assertEquals(2, settings.length()); + + push(new JSONObject().put("v", 1).put("t", 1).put("c", new JSONObject("{\"bom_rqp\":0.25}")).toString()); + Assert.assertEquals(0.25, provider().getBOMRQPercentage(), 0.0); + } + + /** + * Every settings key at once, one key of each blacklist and whitelist pair, at a valid value that + * differs from the default, then every key at an invalid value. + *

+ * Verifies that each value reaches its provider getter and the store, that the request queue size + * is reported as coming from the settings, and that a response made only of invalid values changes + * nothing, neither the resolved values nor the stored settings. + */ + @Test + public void allSettings_resolveFromAResponse_andInvalidValuesNeverOverrideThem() throws IllegalAccessException { + AtomicInteger changes = installCountingModule().changes; + ServerConfigBuilder allKeys = new ServerConfigBuilder().allKeysAtNonDefaultValues(); + server.respondWith(allKeys); + init(TestUtils.getConfigSdkBehaviorSettings()); + + allKeys.validateAgainst(provider()); + Assert.assertTrue(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(1, changes.get()); + JSONObject storedSettings = storedConfig().getJSONObject("c"); + Assert.assertEquals(allKeys.config.keySet(), storedSettings.keySet()); + + ServerConfigBuilder allInvalid = new ServerConfigBuilder(); + for (String key : settingsKeysOfTheModule()) { + allInvalid.set(key, key.equals("tracking") ? -1 : "not a valid value"); + } + push(allInvalid); + + allKeys.validateAgainst(provider()); + Assert.assertEquals(1, changes.get()); + Assert.assertTrue(storedSettings.similar(storedConfig().getJSONObject("c"))); + } + + /** + * The envelope of a response: {@code v}, {@code t} and an object {@code c} are required, any + * other top level key is allowed. + *

+ * Verifies that a response missing one of the three, or with a {@code c} that is not an object, + * is rejected without touching the store, that an empty {@code c} is accepted, that extra top + * level keys are accepted quietly and never stored, so neither {@code lg} nor {@code ct} ever is, + * and that a rejected response after an accepted one keeps the merged settings. + */ + @Test + public void envelope_requiresVersionTimestampAndSettingsObject_andOnlyThoseAreStored() { + List warnings = new CopyOnWriteArrayList<>(); + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (level == Config.LoggingLevel.WARN || level == Config.LoggingLevel.ERROR) { + warnings.add(message); + } + })); + Assert.assertNull(storedConfig()); + + for (String rejected : Arrays.asList("{}", "{'t':2,'c':{'tracking':false}}", "{'v':1,'c':{'tracking':false}}", "{'v':1,'t':2}", + "{'v':1,'t':2,'c':123}", "{'v':1,'t':2,'c':false}", "{'v':1,'t':2,'c':'fdf'}", "{'v':1,'t':2,'c':[]}")) { + push(rejected); + Assert.assertNull("stored after " + rejected, storedConfig()); + Assert.assertTrue(provider().getTrackingEnabled()); + } + + push("{'v':1,'t':2,'c':{}}"); + Assert.assertTrue(new JSONObject("{'v':1,'t':2,'c':{}}").similar(storedConfig())); + + warnings.clear(); + push(new ServerConfigBuilder().version(2).timestamp(3).tracking(false) + .logGatheringOn("gather_id", "ew", 20) + .connectionTest(1) + .topLevelKey("someFutureFeature", "a value this SDK has never heard of")); + + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.GATHERING, provider().getLogGatheringState()); + JSONObject stored = storedConfig(); + Assert.assertEquals(names("v", "t", "c"), stored.keySet()); + Assert.assertEquals(2, stored.getInt("v")); + Assert.assertEquals(3, stored.getLong("t")); + Assert.assertFalse(stored.getJSONObject("c").getBoolean("tracking")); + for (String warning : warnings) { + Assert.assertFalse("warned about a top level key: " + warning, warning.contains("someFutureFeature") || warning.contains("[lg]") || warning.contains("[ct]")); + } + + push("{'t':4,'c':{'tracking':true}}"); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertTrue(stored.similar(storedConfig())); + } + + /** + * The layers of a value: developer configuration, then the provided settings while nothing is + * stored, then the stored settings, then each server response. + *

+ * Verifies each layer overriding the one below it, that the provided settings are stored and + * ignored once something is stored, that a response merges into the stored settings instead of + * replacing them, and that the merge survives {@code stop()} and a new init. + */ + @Test + public void precedence_developerThenProvidedThenStoredThenServer_andTheMergeSurvivesARestart() { + server.pending(); + init(developerConfig()); + + Assert.assertEquals(7, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(50, provider().getRequestQueueMaxSize()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(30, provider().getSessionUpdateInterval()); + Assert.assertEquals(40, provider().getMaxBreadcrumbCount()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getMaxKeyLength()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getUserPropertyCacheLimit()); + Assert.assertTrue(provider().getLoggingEnabled()); + Assert.assertNull(storedConfig()); + Countly.instance().halt(); + + init(developerConfig().setSdkBehaviorSettings(new ServerConfigBuilder().eventQueueSize(11).requestQueueSize(22).build())); + + Assert.assertEquals(11, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(22, provider().getRequestQueueMaxSize()); + Assert.assertTrue(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(30, provider().getSessionUpdateInterval()); + Assert.assertEquals(2, storedConfig().getJSONObject("c").length()); + + push(new ServerConfigBuilder().eventQueueSize(33).breadcrumbLimit(5).logging(false)); + + Assert.assertEquals(33, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(22, provider().getRequestQueueMaxSize()); + Assert.assertEquals(5, provider().getMaxBreadcrumbCount()); + Assert.assertEquals(30, provider().getSessionUpdateInterval()); + Assert.assertFalse(provider().getLoggingEnabled()); + JSONObject storedSettings = storedConfig().getJSONObject("c"); + Assert.assertEquals(names("eqs", "rqs", "lbc", "log"), storedSettings.keySet()); + Countly.instance().stop(); + + server.failing(); + init(developerConfig().setSdkBehaviorSettings(new ServerConfigBuilder().eventQueueSize(99).sessionTracking(false).build())); + + Assert.assertEquals(33, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(22, provider().getRequestQueueMaxSize()); + Assert.assertEquals(5, provider().getMaxBreadcrumbCount()); + Assert.assertFalse(provider().getLoggingEnabled()); + Assert.assertTrue(provider().getSessionTrackingEnabled()); + Assert.assertTrue(storedSettings.similar(storedConfig().getJSONObject("c"))); + } + + /** + * The settings request itself. + *

+ * Verifies that init sends exactly one request, to {@code /o/sdk?} with {@code method=sc} and the + * parameters every request carries, that it goes out even while the settings forbid networking, + * that the configuration module is initialized once and sits at its fixed index ahead of the + * feature modules, and that with settings requests disabled nothing is requested while the + * provided settings still apply. + */ + @Test + public void fetch_requestsTheSettingsAtInitWithTheRequiredParams_andNeverWhenDisabled() { + List initLines = new CopyOnWriteArrayList<>(); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (message.startsWith("[ModuleConfiguration] init,")) { + initLines.add(message); + } + })); + + Assert.assertEquals(1, server.requests.size()); + Assert.assertEquals("/o/sdk?", server.endpoints.get(0)); + Map params = server.requests.get(0); + Assert.assertEquals("sc", params.get("method")); + TestUtils.validateRequiredParams(params); + Assert.assertEquals(TestUtils.SDK_NAME, params.get("sdk_name")); + Assert.assertEquals(TestUtils.SDK_VERSION, params.get("sdk_version")); + Assert.assertTrue(server.networkingFlags.get(0)); + + Assert.assertEquals(1, initLines.size()); + Assert.assertSame(module(), SDKCore.instance.modules.get(-1)); + Assert.assertSame(module(), SDKCore.instance.config.getConfigurationProvider()); + Assert.assertEquals(4L * 60 * 60, module().serverConfigUpdateTimer.getTimerDelaySeconds()); + + Assert.assertFalse(provider().getNetworkingEnabled()); + push(new ServerConfigBuilder().networking(true)); + Assert.assertEquals(2, server.requests.size()); + Assert.assertTrue(server.networkingFlags.get(1)); + Assert.assertTrue(provider().getNetworkingEnabled()); + Countly.instance().halt(); + + server = new ServerConfigResponder(); + init(TestUtils.getConfigSdkBehaviorSettings() + .disableSdkBehaviorSettingsUpdates() + .setSdkBehaviorSettings(new ServerConfigBuilder().tracking(false).build())); + + Assert.assertTrue(server.requests.isEmpty()); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertNull(module().serverConfigUpdateTimer); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.NOT_GATHERING, provider().getLogGatheringState()); + } + + /** + * Backend mode keeps the settings inert. + *

+ * Verifies that with stored and provided settings present, backend mode requests nothing, serves + * the developer configuration, starts no refresh timer, decides against log gathering and leaves + * the stored settings as they are for a later run without backend mode. + */ + @Test + public void backendMode_keepsTheSettingsInert() { + server.respondWith(new ServerConfigBuilder().tracking(false).eventQueueSize(3).requestQueueSize(5)); + init(TestUtils.getConfigSdkBehaviorSettings()); + Assert.assertFalse(provider().getTrackingEnabled()); + JSONObject storedBefore = storedConfig(); + Countly.instance().stop(); + + server = new ServerConfigResponder().respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableBackendMode() + .setEventQueueSizeToSend(9) + .setRequestQueueMaxSize(40) + .setSdkBehaviorSettings(new ServerConfigBuilder().networking(false).build())); + + Assert.assertTrue(server.requests.isEmpty()); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertTrue(provider().getNetworkingEnabled()); + Assert.assertTrue(SDKCore.instance.config.getNetworkingEnabled()); + Assert.assertEquals(9, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(40, provider().getRequestQueueMaxSize()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.NOT_GATHERING, provider().getLogGatheringState()); + Assert.assertNull(module().serverConfigUpdateTimer); + Assert.assertTrue(storedBefore.similar(storedConfig())); + } + + /** + * The consent requirement is applied when the SDK initializes, and only then. + *

+ * Verifies that a {@code cr} arriving while the SDK runs is stored but leaves the running SDK + * without consent gating, that the stored value is in place before the modules are built on the + * next init, so the feature modules wait for consent, and that removing consent stops feature + * modules but never the configuration module, which keeps applying responses. + */ + @Test + public void consentRequirement_appliesAtInitOnly_andConsentRemovalNeverStopsTheConfigurationModule() { + server.respondWith(new ServerConfigBuilder().consentRequired(true)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events)); + + Assert.assertTrue(provider().getConsentRequired()); + Assert.assertFalse(SDKCore.instance.config.requiresConsent()); + Assert.assertTrue(Countly.isTracking(Config.Feature.Events)); + Assert.assertNotNull(Countly.instance().events()); + Assert.assertTrue(storedConfig().getJSONObject("c").getBoolean("cr")); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events)); + + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Assert.assertNull(Countly.instance().events()); + + Countly.onConsent(Config.Feature.Events); + Assert.assertTrue(Countly.isTracking(Config.Feature.Events)); + + ModuleConfiguration configurationModule = module(); + CountlyTimer refreshTimer = configurationModule.serverConfigUpdateTimer; + Assert.assertNotNull(refreshTimer); + + Countly.onConsentRemoval(Config.Feature.Events); + + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Assert.assertSame(configurationModule, SDKCore.instance.modules.get(-1)); + Assert.assertSame(configurationModule, SDKCore.instance.config.getConfigurationProvider()); + Assert.assertTrue(configurationModule.isActive()); + Assert.assertSame(refreshTimer, configurationModule.serverConfigUpdateTimer); + + push(new ServerConfigBuilder().consentRequired(false).tracking(false)); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertFalse(provider().getConsentRequired()); + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + } + + /** + * The global timer follows the session update interval, and the refresh timer follows the + * update interval of the settings. + *

+ * Verifies that a stored {@code sui} starts the global timer at init, which then really ticks every + * second, that a changed {@code sui} restarts the timer and notifies every module once, that a + * response changing nothing restarts nothing and notifies nobody, that another change notifies the + * modules without touching the timer, and that a changed {@code scui} restarts the refresh timer. + */ + @Test + public void timers_followTheResolvedIntervals_andChangesAreDispatchedOnce() throws InterruptedException { + CountingModule counter = installCountingModule(); + AtomicInteger changes = counter.changes; + AtomicInteger ticks = counter.ticks; + + server.respondWith(new ServerConfigBuilder().sessionUpdateInterval(1)); + init(TestUtils.getConfigSdkBehaviorSettings()); + Assert.assertEquals(1, SDKCore.instance.countlyTimer.getTimerDelaySeconds()); + Assert.assertEquals(1, changes.get()); + Countly.instance().stop(); + + changes.set(0); + ticks.set(0); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + + CountlyTimer timerAtInit = SDKCore.instance.countlyTimer; + Assert.assertEquals(1, timerAtInit.getTimerDelaySeconds()); + Assert.assertEquals(0, changes.get()); + Assert.assertTrue("the global timer did not tick every second", waitFor(10_000, () -> ticks.get() >= 2)); + + ServerConfigBuilder slower = new ServerConfigBuilder().sessionUpdateInterval(3); + push(slower); + CountlyTimer restarted = SDKCore.instance.countlyTimer; + Assert.assertNotSame(timerAtInit, restarted); + Assert.assertEquals(3, restarted.getTimerDelaySeconds()); + Assert.assertEquals(1, changes.get()); + + push(slower); + Assert.assertSame(restarted, SDKCore.instance.countlyTimer); + Assert.assertEquals(1, changes.get()); + + push(new ServerConfigBuilder().sessionUpdateInterval(3).eventQueueSize(4)); + Assert.assertSame(restarted, SDKCore.instance.countlyTimer); + Assert.assertEquals(2, changes.get()); + + CountlyTimer refreshTimer = module().serverConfigUpdateTimer; + Assert.assertEquals(4L * 60 * 60, refreshTimer.getTimerDelaySeconds()); + push(new ServerConfigBuilder().serverConfigUpdateInterval(2)); + Assert.assertNotSame(refreshTimer, module().serverConfigUpdateTimer); + Assert.assertEquals(2L * 60 * 60, module().serverConfigUpdateTimer.getTimerDelaySeconds()); + Assert.assertEquals(3, changes.get()); + } + + /** + * {@code cet} covers custom events only, {@code tracking} covers every event. + *

+ * Verifies on the wire that with custom event tracking off a custom event is dropped while a view + * started through the views interface and an internal event still reach the queue, flushed at the + * event queue size the settings resolved, that tracking off then drops internal events too, and + * that custom events are recorded again once both are back on. Networking is off throughout, so + * the request stays on disk to be read. + */ + @Test + public void customEventTracking_off_dropsCustomEventsOnly_whileTrackingGatesEveryEvent() { + server.respondWith(new ServerConfigBuilder().customEventTracking(false).eventQueueSize(2).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events, Config.Feature.Views).setEventQueueSizeToSend(100)); + + Countly.instance().events().recordEvent("custom_event"); + TestUtils.validateEQSize(0); + + Countly.instance().views().startView("home"); + TestUtils.validateEQSize(1); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, TestUtils.getCurrentEQ().get(0).key); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + List sent = TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID); + Assert.assertEquals(2, sent.size()); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, sent.get(0).key); + Assert.assertEquals("[CLY]_star_rating", sent.get(1).key); + TestUtils.validateEQSize(0); + + push(new ServerConfigBuilder().tracking(false)); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Countly.instance().views().startView("settings"); + TestUtils.validateEQSize(0); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().tracking(true).customEventTracking(true)); + Countly.instance().events().recordEvent("custom_event"); + TestUtils.validateEQSize(1); + Assert.assertEquals("custom_event", TestUtils.getCurrentEQ().get(0).key); + } + + /** + * Listing filters and journey trigger sets are published as immutable snapshots. + *

+ * Verifies every filter family and both journey sets as parsed, that a reader's snapshot never + * changes under it, that a response with any whitelist drops every stored blacklist and the + * reverse, that a response with neither keeps the stored filters, and that they survive a restart. + */ + @Test + public void listingFilters_arePublishedAsSnapshots_andOneKindReplacesTheStoredOtherKind() { + Map> perEvent = new LinkedHashMap<>(); + perEvent.put("purchase", names("card")); + server.respondWith(new ServerConfigBuilder() + .eventFilterList(names("blocked_a", "blocked_b"), false) + .segmentationFilterList(names("secret"), false) + .eventSegmentationFilterMap(perEvent, false) + .userPropertyFilterList(names("plan"), true) + .journeyTriggerEvents(names("checkout")) + .journeyTriggerViews(names("paywall"))); + init(TestUtils.getConfigSdkBehaviorSettings()); + + ConfigurationProvider.FilterList> events = provider().getEventFilterList(); + assertFilter(events, false, names("blocked_a", "blocked_b")); + assertFilter(provider().getSegmentationFilterList(), false, names("secret")); + assertFilter(provider().getUserPropertyFilterList(), true, names("plan")); + Assert.assertFalse(provider().getEventSegmentationFilterList().isWhitelist()); + Assert.assertEquals(perEvent, provider().getEventSegmentationFilterList().getFilterList()); + Assert.assertEquals(names("checkout"), provider().getJourneyTriggerEvents()); + Assert.assertEquals(names("paywall"), provider().getJourneyTriggerViews()); + assertUnmodifiable(events.getFilterList()); + assertUnmodifiable(provider().getEventSegmentationFilterList().getFilterList().get("purchase")); + assertUnmodifiable(provider().getJourneyTriggerEvents()); + try { + provider().getEventSegmentationFilterList().getFilterList().put("other", names("x")); + Assert.fail("the event segmentation filter can be changed by a reader"); + } catch (UnsupportedOperationException expected) { + Assert.assertEquals(1, provider().getEventSegmentationFilterList().getFilterList().size()); + } + + push(new ServerConfigBuilder().eventFilterList(names("allowed_event"), true)); + + assertFilter(events, false, names("blocked_a", "blocked_b")); + assertFilter(provider().getEventFilterList(), true, names("allowed_event")); + assertFilter(provider().getSegmentationFilterList(), false, names()); + assertFilter(provider().getUserPropertyFilterList(), true, names("plan")); + Assert.assertTrue(provider().getEventSegmentationFilterList().getFilterList().isEmpty()); + Assert.assertEquals(names("checkout"), provider().getJourneyTriggerEvents()); + Assert.assertEquals(names("ew", "upw", "jte", "jtv"), storedConfig().getJSONObject("c").keySet()); + + push(new ServerConfigBuilder().tracking(false)); + assertFilter(provider().getEventFilterList(), true, names("allowed_event")); + + push(new ServerConfigBuilder().segmentationFilterList(names("secret"), false)); + assertFilter(provider().getEventFilterList(), false, names()); + assertFilter(provider().getUserPropertyFilterList(), false, names()); + assertFilter(provider().getSegmentationFilterList(), false, names("secret")); + Assert.assertEquals(names("jte", "jtv", "tracking", "sb"), storedConfig().getJSONObject("c").keySet()); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertFilter(provider().getSegmentationFilterList(), false, names("secret")); + Assert.assertEquals(names("checkout"), provider().getJourneyTriggerEvents()); + Assert.assertEquals(names("paywall"), provider().getJourneyTriggerViews()); + Assert.assertFalse(provider().getTrackingEnabled()); + } + + /** + * The {@code ct} connection test flag of a live response. + *

+ * Verifies the truthiness table of the flag and that it is removed from the response, then through + * live fetches that a truthy flag reaches the registered listener once, after the settings of the + * same response were applied, with the latency measured from just before the request to its + * response, that a falsy flag does not, that a response whose settings are rejected can still ask, + * and that the flag is never stored. + */ + @Test + public void connectionTestFlag_isReadByValue_strippedFromTheResponse_andHandedOnWithTheFetchLatency() { + Object[][] table = { + { "true", true }, { "false", false }, { "1", true }, { "0", false }, { "0.0", false }, { "2.5", true }, { "-1", true }, + { "\"1\"", true }, { "\"0\"", false }, { "\"false\"", false }, { "\"FALSE\"", false }, { "\" false \"", false }, + { "\"\"", false }, { "\" \"", false }, { "\"yes\"", true }, { "\"true\"", true }, { "null", false }, + { "{}", true }, { "[]", true }, { "{\"a\":1}", true } + }; + for (Object[] row : table) { + JSONObject response = new JSONObject("{\"v\":1,\"t\":1,\"c\":{},\"ct\":" + row[0] + "}"); + Assert.assertEquals("ct = " + row[0], row[1], ModuleConfiguration.extractConnectionTestFlag(response)); + Assert.assertFalse("ct = " + row[0] + " was left in the response", response.has("ct")); + } + Assert.assertFalse(ModuleConfiguration.extractConnectionTestFlag(new JSONObject("{\"v\":1,\"t\":1,\"c\":{}}"))); + Assert.assertFalse(ModuleConfiguration.extractConnectionTestFlag(null)); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + List latencies = new CopyOnWriteArrayList<>(); + List trackingWhenNotified = new CopyOnWriteArrayList<>(); + module().setConnectionTestListener(fetchLatencyMs -> { + latencies.add(fetchLatencyMs); + trackingWhenNotified.add(provider().getTrackingEnabled()); + }); + + server.delayedBy(80); + push(new ServerConfigBuilder().tracking(false).connectionTest(1)); + Assert.assertEquals(1, latencies.size()); + Assert.assertTrue("latency " + latencies.get(0), latencies.get(0) >= 70); + Assert.assertEquals(Arrays.asList(false), trackingWhenNotified); + + server.delayedBy(0); + push(new ServerConfigBuilder().connectionTest(0)); + push(new ServerConfigBuilder().connectionTest("false")); + push(new ServerConfigBuilder()); + Assert.assertEquals(1, latencies.size()); + + push("{\"ct\":true}"); + Assert.assertEquals(2, latencies.size()); + Assert.assertFalse(storedConfig().has("ct")); + + module().setConnectionTestListener(null); + push(new ServerConfigBuilder().connectionTest(1).tracking(true)); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertEquals(2, latencies.size()); + } + + /** + * The {@code lg} directive of a live response turns log gathering on and off. + *

+ * Verifies that a directive with a gather id arms gathering with its levels and batch size and is + * never stored, that {@code "e":false} and a response without any directive both decide off and + * drop the gather id, and that a failed fetch after a decision changes nothing. + */ + @Test + public void logGatheringDirective_enabledThenDisabled_isAppliedBothWays() { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_id_3", "ewd", 40)); + init(TestUtils.getConfigSdkBehaviorSettings()); + + assertGathering("gather_id_3", "ewd", 40); + Assert.assertFalse(storedConfig().has("lg")); + + push(new ServerConfigBuilder().logGatheringOff()); + assertNotGathering(); + + push(new ServerConfigBuilder().logGatheringOn("gather_id_3", "ewd", 40)); + assertGathering("gather_id_3", "ewd", 40); + + push(new ServerConfigBuilder().tracking(false)); + assertNotGathering(); + Assert.assertFalse(provider().getTrackingEnabled()); + + push(new ServerConfigBuilder().logGatheringOn("gather_id_4", "e", 20)); + server.failing(); + module().fetchConfigFromServer(SDKCore.instance.config); + assertGathering("gather_id_4", "e", 20); + } + + /** + * A directive that enables gathering without a usable gather id, or with {@code e} that is not a + * boolean {@code true}, decides off. A working directive is applied between the cases, so no + * assertion passes because the state was already off. + */ + @Test + public void logGatheringDirective_withoutUsableGatherId_doesNotEnableGathering() { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_id_4", "ew", 20)); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertGathering("gather_id_4", "ew", 20); + + List unusable = Arrays.asList( + new ServerConfigBuilder().logGatheringDirective(true, null, "ew", 20), + new ServerConfigBuilder().logGatheringDirective(true, " ", "ew", 20), + new ServerConfigBuilder().logGatheringDirective(true, 42, "ew", 20), + new ServerConfigBuilder().logGatheringDirective("true", "gather_id_4", "ew", 20), + new ServerConfigBuilder().logGatheringDirective(1, "gather_id_4", "ew", 20), + new ServerConfigBuilder().topLevelKey("lg", "on")); + for (ServerConfigBuilder directive : unusable) { + push(directive); + assertNotGathering(); + + push(new ServerConfigBuilder().logGatheringOn("gather_id_4", "ew", 20)); + assertGathering("gather_id_4", "ew", 20); + } + + push(new ServerConfigBuilder().logGatheringOn(" gather_id_5 ", "ew", 20)); + assertGathering("gather_id_5", "ew", 20); + } + + /** + * The levels and the batch size of a directive are sanitized: unknown level characters go, the + * rest is lower cased and deduplicated in the order sent, nothing usable falls back to every level, + * and the batch size is clamped into [10, 500] or falls back to 100 when it is not a number. + */ + @Test + public void logGatheringDirective_levelsAndBatchSize_areSanitized() { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_id_5", "xyz!", 100)); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertGathering("gather_id_5", ModuleConfiguration.logGatheringAllLevels, 100); + + Object[][] steps = { + { "ew", 100, "ew", 100 }, + { null, 100, ModuleConfiguration.logGatheringAllLevels, 100 }, + { "ew", 100, "ew", 100 }, + { 5, 100, ModuleConfiguration.logGatheringAllLevels, 100 }, + { "EWID", 100, "ewid", 100 }, + { "wweeww", 100, "we", 100 }, + { "vq d", 100, "vd", 100 }, + { "ew", 1, "ew", ModuleConfiguration.logGatheringMinBatchSize }, + { "ew", 9999, "ew", ModuleConfiguration.logGatheringMaxBufferedLines }, + { "ew", 250, "ew", 250 }, + { "ew", "250", "ew", ModuleConfiguration.logGatheringDefaultBatchSize }, + { "ew", 250, "ew", 250 }, + { "ew", null, "ew", ModuleConfiguration.logGatheringDefaultBatchSize }, + { "ew", 37.9, "ew", 37 } + }; + for (Object[] step : steps) { + push(new ServerConfigBuilder().logGatheringDirective(true, "gather_id_5", step[0], step[1])); + assertGathering("gather_id_5", (String) step[2], (Integer) step[3]); + } + } + + /** + * Only a live response decides log gathering. + *

+ * Verifies that a stored directive, even an armed one, leaves the state undecided and is dropped + * from the stored copy, that a stored configuration without one also stays undecided, that a + * failed fetch decides off while undecided, and that disabled settings requests decide off at + * init even with an armed directive in the provided settings. + */ + @Test + public void logGathering_isUndecidedUntilALiveResponse_failedFetchAndDisabledRequestsDecideOff() { + TestUtils.writeToFile(SDKStorage.JSON_FILE_NAME, new JSONObject() + .put("sc", new ServerConfigBuilder().tracking(false).logGatheringOn("stored_gather_id", "ewidv", 40).build()).toString()); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.UNDECIDED, provider().getLogGatheringState()); + Assert.assertNull(provider().getLogGatheringId()); + Assert.assertEquals(ModuleConfiguration.logGatheringAllLevels, provider().getLogGatheringLevels()); + Assert.assertEquals(ModuleConfiguration.logGatheringDefaultBatchSize, provider().getLogGatheringBatchSize()); + Assert.assertFalse(storedConfig().has("lg")); + Countly.instance().stop(); + + init(TestUtils.getConfigSdkBehaviorSettings()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.UNDECIDED, provider().getLogGatheringState()); + Countly.instance().stop(); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertNotGathering(); + Countly.instance().halt(); + + server = new ServerConfigResponder(); + init(TestUtils.getConfigSdkBehaviorSettings() + .disableSdkBehaviorSettingsUpdates() + .setSdkBehaviorSettings(new ServerConfigBuilder().logGatheringOn("provided_gather_id", "ew", 20).build())); + assertNotGathering(); + Assert.assertTrue(server.requests.isEmpty()); + } + + /** + * A queue held back by {@code networking: false} drains as soon as a response allows networking + * again, without waiting for another request or for the global timer. + *

+ * Verifies against a local server that nothing is sent while networking is off and that both held + * requests arrive once it is back on. + */ + @Test + public void networking_turningBackOn_drainsTheRequestsItHeldBack() throws Exception { + AtomicInteger received = new AtomicInteger(); + HttpServer httpServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + httpServer.createContext("/", exchange -> { + if (exchange.getRequestURI().getPath().startsWith("/i")) { + received.incrementAndGet(); + } + byte[] body = "{\"result\":\"Success\"}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(body); + } + }); + httpServer.start(); + + try { + Config config = new Config("http://localhost:" + httpServer.getAddress().getPort(), TestUtils.SERVER_APP_KEY, TestUtils.getTestSDirectory()) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setSdkBehaviorSettings(new ServerConfigBuilder().networking(false).build()); + server.pending(); + init(config); + + Countly.instance().events().recordEvent("held_1"); + Countly.instance().events().recordEvent("held_2"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + //a queue that ignored the switch would have sent the first request well within this + Thread.sleep(1000); + Assert.assertEquals(0, received.get()); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().networking(true)); + + Assert.assertTrue("the held requests never drained", waitFor(30_000, () -> received.get() >= 2 && TestUtils.getCurrentRQ().length == 0)); + Assert.assertEquals(2, received.get()); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + // endregion + // region helpers + + /** + * Stands in for the settings endpoint of the server: records every immediate request the SDK + * makes and answers it on the calling thread with the response set at that moment. + */ + static final class ServerConfigResponder implements ImmediateRequestGenerator { + final List> requests = new CopyOnWriteArrayList<>(); + final List endpoints = new CopyOnWriteArrayList<>(); + final List networkingFlags = new CopyOnWriteArrayList<>(); + private volatile String response = null; + private volatile boolean answering = true; + private volatile long delayMs = 0; + + /** + * Answers every request with the given payload. + * + * @param builder the payload + * @return this responder + */ + ServerConfigResponder respondWith(ServerConfigBuilder builder) { + return respondWith(builder.build()); + } + + /** + * Answers every request with the given JSON text. + * + * @param json the response + * @return this responder + */ + ServerConfigResponder respondWith(String json) { + response = json; + answering = true; + return this; + } + + /** + * Answers every request as a failed one. + * + * @return this responder + */ + ServerConfigResponder failing() { + response = null; + answering = true; + return this; + } + + /** + * Never answers, as a server that has not answered yet. + * + * @return this responder + */ + ServerConfigResponder pending() { + answering = false; + return this; + } + + /** + * Waits before answering. + * + * @param delayMs how long, in milliseconds + * @return this responder + */ + ServerConfigResponder delayedBy(long delayMs) { + this.delayMs = delayMs; + return this; + } + + /** + * A request maker that records the request and answers it. + * + * @return the request maker + */ + @Override + public ImmediateRequestI createImmediateRequestMaker() { + return (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> { + requests.add(parseParams(requestData)); + endpoints.add(customEndpoint); + networkingFlags.add(networkingIsEnabled); + if (!answering) { + return; + } + if (delayMs > 0) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + String current = response; + callback.callback(current == null ? null : new JSONObject(current)); + }; + } + + /** + * Decodes url encoded request parameters. + * + * @param requestData the parameters + * @return the parameters by name + */ + private static Map parseParams(String requestData) { + Map params = new HashMap<>(); + for (String pair : requestData.split("&")) { + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return params; + } + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + push(builder.build()); + } + + /** + * Answers the next settings request with the given JSON text and makes the SDK fetch. + * + * @param json the response + */ + private void push(String json) { + server.respondWith(json); + module().fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The configuration module of the running SDK. + * + * @return the module + */ + private static ModuleConfiguration module() { + return SDKCore.instance.module(ModuleConfiguration.class); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings as stored on disk. + * + * @return the stored settings, {@code null} when nothing is stored + */ + private static JSONObject storedConfig() { + String stored = TestUtils.readJsonFile(SDKStorage.JSON_FILE_NAME).optString("sc", null); + return stored == null ? null : new JSONObject(stored); + } + + /** + * A developer configuration that sets every value the precedence test follows. + * + * @return the configuration + */ + private static Config developerConfig() { + return TestUtils.getConfigSdkBehaviorSettings() + .setEventQueueSizeToSend(7) + .setRequestQueueMaxSize(50) + .setUpdateSessionTimerDelay(30) + .setMaxBreadcrumbCount(40) + .setLoggingLevel(Config.LoggingLevel.ERROR); + } + + /** + * A test module that counts the ticks of the global timer and the configuration changes the SDK + * dispatches to every module. + */ + private static final class CountingModule extends ModuleBase { + final AtomicInteger changes = new AtomicInteger(); + final AtomicInteger ticks = new AtomicInteger(); + + /** + * Counts a tick of the global timer. + */ + @Override + protected void onTimer() { + ticks.incrementAndGet(); + } + + /** + * Counts a dispatched configuration change. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + changes.incrementAndGet(); + } + } + + /** + * Installs a {@link CountingModule} for the next init. + * + * @return the module + */ + private static CountingModule installCountingModule() { + CountingModule module = new CountingModule(); + SDKCore.testDummyModule = module; + return module; + } + + /** + * Every settings key the module declares, read off its {@code keyR} constants. + * + * @return the settings keys, without the top level keys + */ + private static List settingsKeysOfTheModule() throws IllegalAccessException { + List keys = new ArrayList<>(); + for (Field field : ModuleConfiguration.class.getDeclaredFields()) { + if (field.getName().startsWith("keyR") && Modifier.isStatic(field.getModifiers())) { + String key = (String) field.get(null); + if (!TOP_LEVEL_KEYS.contains(key)) { + keys.add(key); + } + } + } + return keys; + } + + /** + * Runs a settings object through the validation of a response. + * + * @param module the module that validates + * @param settings the settings object, as JSON text + * @return the settings left after validation + */ + private static JSONObject validated(ModuleConfiguration module, String settings) { + JSONObject envelope = new JSONObject("{\"v\":1,\"t\":1,\"c\":" + settings + "}"); + Assert.assertTrue(module.validateServerConfig(envelope)); + return envelope.getJSONObject("c"); + } + + /** + * Asserts whether validation keeps one setting. + * + * @param module the module that validates + * @param key the settings key + * @param literal the value, as JSON text + * @param kept whether the setting must be kept + */ + private static void assertKeptByValidation(ModuleConfiguration module, String key, String literal, boolean kept) { + JSONObject settings = validated(module, "{\"" + key + "\":" + literal + "}"); + Assert.assertEquals("[" + key + "] = " + literal, kept, settings.has(key)); + } + + /** + * Asserts the kind and the names of a filter. + * + * @param filter the filter + * @param isWhitelist the expected kind + * @param expected the expected names + */ + private static void assertFilter(ConfigurationProvider.FilterList> filter, boolean isWhitelist, Set expected) { + Assert.assertEquals(isWhitelist, filter.isWhitelist()); + Assert.assertEquals(expected, filter.getFilterList()); + } + + /** + * Asserts that a reader cannot change a collection. + * + * @param collection the collection + */ + private static void assertUnmodifiable(Collection collection) { + try { + collection.add("added_by_a_reader"); + Assert.fail("a reader can change " + collection); + } catch (UnsupportedOperationException expected) { + Assert.assertFalse(collection.contains("added_by_a_reader")); + } + } + + /** + * Asserts that log gathering is on with the given directive. + * + * @param gatherId the gather id + * @param levels the levels + * @param batchSize the batch size + */ + private static void assertGathering(String gatherId, String levels, int batchSize) { + Assert.assertEquals(ConfigurationProvider.LogGatheringState.GATHERING, provider().getLogGatheringState()); + Assert.assertEquals(gatherId, provider().getLogGatheringId()); + Assert.assertEquals(levels, provider().getLogGatheringLevels()); + Assert.assertEquals(batchSize, provider().getLogGatheringBatchSize()); + } + + /** + * Asserts that log gathering was decided off and nothing of a previous directive is left. + */ + private static void assertNotGathering() { + Assert.assertEquals(ConfigurationProvider.LogGatheringState.NOT_GATHERING, provider().getLogGatheringState()); + Assert.assertNull(provider().getLogGatheringId()); + Assert.assertEquals(ModuleConfiguration.logGatheringAllLevels, provider().getLogGatheringLevels()); + Assert.assertEquals(ModuleConfiguration.logGatheringDefaultBatchSize, provider().getLogGatheringBatchSize()); + } + + /** + * Polls until the condition holds or the budget runs out. + * + * @param timeoutMs the budget + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitFor(long timeoutMs, BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(50); + } + return condition.getAsBoolean(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java index 7dab91b7..f03adca9 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java @@ -94,7 +94,9 @@ private Config configForLocalServer() { .setLoggingLevel(Config.LoggingLevel.VERBOSE) .setDeviceIdStrategy(Config.DeviceIdStrategy.UUID) .enableFeatures(Config.Feature.Events, Config.Feature.Sessions) - .setEventQueueSizeToSend(1); + .setEventQueueSizeToSend(1) + // request #1 can be held past the backoff's accepted timeout, and a backoff would hold the backlog past the drain timeout + .disableBackoffMechanism(); } /** diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ServerConfigBuilder.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ServerConfigBuilder.java new file mode 100644 index 00000000..745e8a2c --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ServerConfigBuilder.java @@ -0,0 +1,709 @@ +package ly.count.sdk.java.internal; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import org.json.JSONObject; +import org.junit.Assert; + +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGBatchSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGEnabled; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGId; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGLevels; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRAutomaticCrashReporting; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRAutomaticSessionTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRAutomaticViewTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMAcceptedTimeout; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMDuration; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMRQPercentage; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMRequestAge; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBackoffMechanism; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRConfig; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRConnectionTest; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRConsentRequired; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRContentZoneInterval; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRCrashReporting; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRCustomEventTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRDropOldRequestTime; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREnterContentZone; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventQueueSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventSegmentationBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventSegmentationWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRJourneyTriggerEvents; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRJourneyTriggerViews; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitBreadcrumb; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitKeyLength; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitSegValues; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitTraceLength; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitTraceLine; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitValueSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLocationTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLogGathering; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLogging; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRNetworking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRRefreshContentZone; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRReqQueueSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSegmentationBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSegmentationWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRServerConfigUpdateInterval; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSessionTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSessionUpdateInterval; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRTimestamp; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRUserPropertyBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRUserPropertyCacheLimit; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRUserPropertyWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRVersion; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRViewTracking; + +/** + * Builds SDK behavior settings payloads, {@code {"v":..,"t":..,"c":{..}}} plus any sibling of + * {@code c}, and checks a {@link ConfigurationProvider} against the settings it holds. + */ +class ServerConfigBuilder { + final Map config = new LinkedHashMap<>(); + /** + * Keys that sit next to {@code c}, not inside it: the {@code lg} directive, the {@code ct} + * flag and anything a newer server may add. + */ + final Map topLevelKeys = new LinkedHashMap<>(); + private Object timestamp = System.currentTimeMillis(); + private Object version = 1; + + /** + * Sets the {@code t} value. + * + * @param timestamp the value + * @return this builder + */ + ServerConfigBuilder timestamp(Object timestamp) { + this.timestamp = timestamp; + return this; + } + + /** + * Sets the {@code v} value. + * + * @param version the value + * @return this builder + */ + ServerConfigBuilder version(Object version) { + this.version = version; + return this; + } + + /** + * Sets any setting to any value, including the ones the SDK has to reject. + * + * @param key the settings key + * @param value the value + * @return this builder + */ + ServerConfigBuilder set(String key, Object value) { + config.put(key, value); + return this; + } + + /** + * @param enabled {@code tracking} + * @return this builder + */ + ServerConfigBuilder tracking(boolean enabled) { + return set(keyRTracking, enabled); + } + + /** + * @param enabled {@code networking} + * @return this builder + */ + ServerConfigBuilder networking(boolean enabled) { + return set(keyRNetworking, enabled); + } + + /** + * @param enabled {@code st} + * @return this builder + */ + ServerConfigBuilder sessionTracking(boolean enabled) { + return set(keyRSessionTracking, enabled); + } + + /** + * @param enabled {@code vt} + * @return this builder + */ + ServerConfigBuilder viewTracking(boolean enabled) { + return set(keyRViewTracking, enabled); + } + + /** + * @param enabled {@code cet} + * @return this builder + */ + ServerConfigBuilder customEventTracking(boolean enabled) { + return set(keyRCustomEventTracking, enabled); + } + + /** + * @param enabled {@code ecz} + * @return this builder + */ + ServerConfigBuilder contentZone(boolean enabled) { + return set(keyREnterContentZone, enabled); + } + + /** + * @param enabled {@code crt} + * @return this builder + */ + ServerConfigBuilder crashReporting(boolean enabled) { + return set(keyRCrashReporting, enabled); + } + + /** + * @param enabled {@code ast} + * @return this builder + */ + ServerConfigBuilder automaticSessionTracking(boolean enabled) { + return set(keyRAutomaticSessionTracking, enabled); + } + + /** + * @param enabled {@code avt} + * @return this builder + */ + ServerConfigBuilder automaticViewTracking(boolean enabled) { + return set(keyRAutomaticViewTracking, enabled); + } + + /** + * @param enabled {@code acr} + * @return this builder + */ + ServerConfigBuilder automaticCrashReporting(boolean enabled) { + return set(keyRAutomaticCrashReporting, enabled); + } + + /** + * @param enabled {@code lt} + * @return this builder + */ + ServerConfigBuilder locationTracking(boolean enabled) { + return set(keyRLocationTracking, enabled); + } + + /** + * @param enabled {@code rcz} + * @return this builder + */ + ServerConfigBuilder refreshContentZone(boolean enabled) { + return set(keyRRefreshContentZone, enabled); + } + + /** + * @param enabled {@code bom} + * @return this builder + */ + ServerConfigBuilder backoffMechanism(boolean enabled) { + return set(keyRBackoffMechanism, enabled); + } + + /** + * @param enabled {@code log} + * @return this builder + */ + ServerConfigBuilder logging(boolean enabled) { + return set(keyRLogging, enabled); + } + + /** + * @param required {@code cr} + * @return this builder + */ + ServerConfigBuilder consentRequired(boolean required) { + return set(keyRConsentRequired, required); + } + + /** + * @param hours {@code scui} + * @return this builder + */ + ServerConfigBuilder serverConfigUpdateInterval(int hours) { + return set(keyRServerConfigUpdateInterval, hours); + } + + /** + * @param size {@code rqs} + * @return this builder + */ + ServerConfigBuilder requestQueueSize(int size) { + return set(keyRReqQueueSize, size); + } + + /** + * @param size {@code eqs} + * @return this builder + */ + ServerConfigBuilder eventQueueSize(int size) { + return set(keyREventQueueSize, size); + } + + /** + * @param seconds {@code sui} + * @return this builder + */ + ServerConfigBuilder sessionUpdateInterval(int seconds) { + return set(keyRSessionUpdateInterval, seconds); + } + + /** + * @param limit {@code lkl} + * @return this builder + */ + ServerConfigBuilder keyLengthLimit(int limit) { + return set(keyRLimitKeyLength, limit); + } + + /** + * @param limit {@code lvs} + * @return this builder + */ + ServerConfigBuilder valueSizeLimit(int limit) { + return set(keyRLimitValueSize, limit); + } + + /** + * @param limit {@code lsv} + * @return this builder + */ + ServerConfigBuilder segmentationValuesLimit(int limit) { + return set(keyRLimitSegValues, limit); + } + + /** + * @param limit {@code lbc} + * @return this builder + */ + ServerConfigBuilder breadcrumbLimit(int limit) { + return set(keyRLimitBreadcrumb, limit); + } + + /** + * @param limit {@code ltlpt} + * @return this builder + */ + ServerConfigBuilder traceLinesLimit(int limit) { + return set(keyRLimitTraceLine, limit); + } + + /** + * @param limit {@code ltl} + * @return this builder + */ + ServerConfigBuilder traceLengthLimit(int limit) { + return set(keyRLimitTraceLength, limit); + } + + /** + * @param limit {@code upcl} + * @return this builder + */ + ServerConfigBuilder userPropertyCacheLimit(int limit) { + return set(keyRUserPropertyCacheLimit, limit); + } + + /** + * @param seconds {@code bom_at} + * @return this builder + */ + ServerConfigBuilder backoffAcceptedTimeout(int seconds) { + return set(keyRBOMAcceptedTimeout, seconds); + } + + /** + * @param share {@code bom_rqp} + * @return this builder + */ + ServerConfigBuilder backoffRequestQueuePercentage(double share) { + return set(keyRBOMRQPercentage, share); + } + + /** + * @param hours {@code bom_ra} + * @return this builder + */ + ServerConfigBuilder backoffRequestAge(int hours) { + return set(keyRBOMRequestAge, hours); + } + + /** + * @param seconds {@code bom_d} + * @return this builder + */ + ServerConfigBuilder backoffDuration(int seconds) { + return set(keyRBOMDuration, seconds); + } + + /** + * @param hours {@code dort} + * @return this builder + */ + ServerConfigBuilder dropOldRequestTime(int hours) { + return set(keyRDropOldRequestTime, hours); + } + + /** + * @param seconds {@code czi} + * @return this builder + */ + ServerConfigBuilder contentZoneInterval(int seconds) { + return set(keyRContentZoneInterval, seconds); + } + + /** + * Sets {@code eb} or {@code ew}, dropping the other one. + * + * @param names the event keys + * @param isWhitelist {@code true} for {@code ew} + * @return this builder + */ + ServerConfigBuilder eventFilterList(Set names, boolean isWhitelist) { + return filter(keyREventBlacklist, keyREventWhitelist, names, isWhitelist); + } + + /** + * Sets {@code upb} or {@code upw}, dropping the other one. + * + * @param names the user property keys + * @param isWhitelist {@code true} for {@code upw} + * @return this builder + */ + ServerConfigBuilder userPropertyFilterList(Set names, boolean isWhitelist) { + return filter(keyRUserPropertyBlacklist, keyRUserPropertyWhitelist, names, isWhitelist); + } + + /** + * Sets {@code sb} or {@code sw}, dropping the other one. + * + * @param names the segmentation keys + * @param isWhitelist {@code true} for {@code sw} + * @return this builder + */ + ServerConfigBuilder segmentationFilterList(Set names, boolean isWhitelist) { + return filter(keyRSegmentationBlacklist, keyRSegmentationWhitelist, names, isWhitelist); + } + + /** + * Sets {@code esb} or {@code esw}, dropping the other one. + * + * @param namesPerEvent the segmentation keys of each event key + * @param isWhitelist {@code true} for {@code esw} + * @return this builder + */ + ServerConfigBuilder eventSegmentationFilterMap(Map> namesPerEvent, boolean isWhitelist) { + config.remove(isWhitelist ? keyREventSegmentationBlacklist : keyREventSegmentationWhitelist); + return set(isWhitelist ? keyREventSegmentationWhitelist : keyREventSegmentationBlacklist, namesPerEvent); + } + + /** + * @param eventKeys {@code jte} + * @return this builder + */ + ServerConfigBuilder journeyTriggerEvents(Set eventKeys) { + return set(keyRJourneyTriggerEvents, eventKeys); + } + + /** + * @param viewNames {@code jtv} + * @return this builder + */ + ServerConfigBuilder journeyTriggerViews(Set viewNames) { + return set(keyRJourneyTriggerViews, viewNames); + } + + /** + * Sets one of a blacklist and whitelist pair, dropping the other one. + * + * @param blacklistKey the blacklist key + * @param whitelistKey the whitelist key + * @param names the names + * @param isWhitelist whether to set the whitelist + * @return this builder + */ + private ServerConfigBuilder filter(String blacklistKey, String whitelistKey, Set names, boolean isWhitelist) { + config.remove(isWhitelist ? blacklistKey : whitelistKey); + return set(isWhitelist ? whitelistKey : blacklistKey, names); + } + + /** + * Sets any key next to {@code c}. + * + * @param key the top level key + * @param value the value + * @return this builder + */ + ServerConfigBuilder topLevelKey(String key, Object value) { + topLevelKeys.put(key, value); + return this; + } + + /** + * Sets the {@code ct} connection test flag. + * + * @param value the flag, of any type + * @return this builder + */ + ServerConfigBuilder connectionTest(Object value) { + return topLevelKey(keyRConnectionTest, value); + } + + /** + * A log gathering directive that turns gathering off, {@code {"e":false}}. + * + * @return this builder + */ + ServerConfigBuilder logGatheringOff() { + return logGatheringDirective(false, null, null, null); + } + + /** + * A log gathering directive with every field filled in. + * + * @param gatherId {@code i} + * @param levels {@code l} + * @param batchSize {@code b} + * @return this builder + */ + ServerConfigBuilder logGatheringOn(String gatherId, String levels, int batchSize) { + return logGatheringDirective(true, gatherId, levels, batchSize); + } + + /** + * A log gathering directive from raw values, so malformed ones can be built too. A {@code null} + * value is left out rather than written as JSON null. + * + * @param enabled {@code e} + * @param gatherId {@code i} + * @param levels {@code l} + * @param batchSize {@code b} + * @return this builder + */ + ServerConfigBuilder logGatheringDirective(Object enabled, Object gatherId, Object levels, Object batchSize) { + JSONObject directive = new JSONObject(); + putIfNotNull(directive, keyLGEnabled, enabled); + putIfNotNull(directive, keyLGId, gatherId); + putIfNotNull(directive, keyLGLevels, levels); + putIfNotNull(directive, keyLGBatchSize, batchSize); + return topLevelKey(keyRLogGathering, directive); + } + + /** + * Puts a value unless it is {@code null}. + * + * @param target the object to put into + * @param key the key + * @param value the value + */ + private static void putIfNotNull(JSONObject target, String key, Object value) { + if (value != null) { + target.put(key, value); + } + } + + /** + * Every setting at a valid value that differs from what the SDK uses by default. + * + * @return this builder + */ + ServerConfigBuilder allKeysAtNonDefaultValues() { + tracking(false).networking(false).sessionTracking(false).viewTracking(false).customEventTracking(false) + .contentZone(true).crashReporting(false).automaticSessionTracking(true).automaticViewTracking(true) + .automaticCrashReporting(false).locationTracking(false).refreshContentZone(false).backoffMechanism(false) + .logging(true).consentRequired(true); + + serverConfigUpdateInterval(8).requestQueueSize(2000).eventQueueSize(200).sessionUpdateInterval(120) + .keyLengthLimit(89).valueSizeLimit(43).segmentationValuesLimit(25).breadcrumbLimit(90) + .traceLinesLimit(89).traceLengthLimit(78).userPropertyCacheLimit(67) + .backoffAcceptedTimeout(12).backoffRequestQueuePercentage(0.25).backoffRequestAge(36).backoffDuration(90) + .dropOldRequestTime(5).contentZoneInterval(60); + + eventFilterList(names("blocked_event"), false); + userPropertyFilterList(names("allowed_property"), true); + segmentationFilterList(names("blocked_segment"), false); + Map> perEvent = new LinkedHashMap<>(); + perEvent.put("purchase", names("card_number")); + eventSegmentationFilterMap(perEvent, false); + journeyTriggerEvents(names("journey_event")); + return journeyTriggerViews(names("journey_view")); + } + + /** + * A set of names in the given order. + * + * @param names the names + * @return the set + */ + static Set names(String... names) { + return new LinkedHashSet<>(Arrays.asList(names)); + } + + /** + * The payload as the server sends it. + * + * @return the JSON text + */ + String build() { + return buildJson().toString(); + } + + /** + * The payload as a JSON object. + * + * @return the payload + */ + JSONObject buildJson() { + JSONObject json = new JSONObject(); + json.put(keyRVersion, version); + json.put(keyRTimestamp, timestamp); + json.put(keyRConfig, new JSONObject(config)); + for (Map.Entry entry : topLevelKeys.entrySet()) { + json.put(entry.getKey(), entry.getValue()); + } + return json; + } + + /** + * Asserts that the provider serves every setting this builder holds. + * + * @param provider the provider to check + */ + @SuppressWarnings("unchecked") + void validateAgainst(ConfigurationProvider provider) { + for (Map.Entry entry : config.entrySet()) { + String key = entry.getKey(); + Object expected = entry.getValue(); + switch (key) { + case keyREventBlacklist: + case keyREventWhitelist: + assertFilter(key, (Set) expected, keyREventWhitelist.equals(key), provider.getEventFilterList()); + break; + case keyRUserPropertyBlacklist: + case keyRUserPropertyWhitelist: + assertFilter(key, (Set) expected, keyRUserPropertyWhitelist.equals(key), provider.getUserPropertyFilterList()); + break; + case keyRSegmentationBlacklist: + case keyRSegmentationWhitelist: + assertFilter(key, (Set) expected, keyRSegmentationWhitelist.equals(key), provider.getSegmentationFilterList()); + break; + case keyREventSegmentationBlacklist: + case keyREventSegmentationWhitelist: + ConfigurationProvider.FilterList>> perEvent = provider.getEventSegmentationFilterList(); + Assert.assertEquals("kind of '" + key + "'", keyREventSegmentationWhitelist.equals(key), perEvent.isWhitelist()); + Assert.assertEquals("names of '" + key + "'", expected, perEvent.getFilterList()); + break; + default: + Assert.assertEquals("value of '" + key + "'", expected, valueOf(provider, key)); + break; + } + } + } + + /** + * Asserts one names filter. + * + * @param key the settings key + * @param expected the names + * @param isWhitelist the expected kind + * @param actual the filter the provider serves + */ + private static void assertFilter(String key, Set expected, boolean isWhitelist, ConfigurationProvider.FilterList> actual) { + Assert.assertEquals("kind of '" + key + "'", isWhitelist, actual.isWhitelist()); + Assert.assertEquals("names of '" + key + "'", expected, actual.getFilterList()); + } + + /** + * The value a provider serves for a setting that is not a listing filter. + * + * @param provider the provider + * @param key the settings key + * @return the served value, boxed + */ + static Object valueOf(ConfigurationProvider provider, String key) { + switch (key) { + case keyRTracking: + return provider.getTrackingEnabled(); + case keyRNetworking: + return provider.getNetworkingEnabled(); + case keyRSessionTracking: + return provider.getSessionTrackingEnabled(); + case keyRViewTracking: + return provider.getViewTrackingEnabled(); + case keyRCustomEventTracking: + return provider.getCustomEventTrackingEnabled(); + case keyREnterContentZone: + return provider.getContentZoneEnabled(); + case keyRCrashReporting: + return provider.getCrashReportingEnabled(); + case keyRAutomaticSessionTracking: + return provider.getAutomaticSessionTrackingEnabled(); + case keyRAutomaticViewTracking: + return provider.getAutomaticViewTrackingEnabled(); + case keyRAutomaticCrashReporting: + return provider.getAutomaticCrashReportingEnabled(); + case keyRLocationTracking: + return provider.getLocationTrackingEnabled(); + case keyRRefreshContentZone: + return provider.getRefreshContentZoneEnabled(); + case keyRBackoffMechanism: + return provider.getBOMEnabled(); + case keyRLogging: + return provider.getLoggingEnabled(); + case keyRConsentRequired: + return provider.getConsentRequired(); + case keyRServerConfigUpdateInterval: + return provider.getServerConfigUpdateInterval(); + case keyRReqQueueSize: + return provider.getRequestQueueMaxSize(); + case keyREventQueueSize: + return provider.getEventQueueSizeThreshold(); + case keyRSessionUpdateInterval: + return provider.getSessionUpdateInterval(); + case keyRLimitKeyLength: + return provider.getMaxKeyLength(); + case keyRLimitValueSize: + return provider.getMaxValueSize(); + case keyRLimitSegValues: + return provider.getMaxSegmentationValues(); + case keyRLimitBreadcrumb: + return provider.getMaxBreadcrumbCount(); + case keyRLimitTraceLine: + return provider.getMaxStackTraceLinesPerThread(); + case keyRLimitTraceLength: + return provider.getMaxStackTraceLineLength(); + case keyRUserPropertyCacheLimit: + return provider.getUserPropertyCacheLimit(); + case keyRBOMAcceptedTimeout: + return provider.getBOMAcceptedTimeoutSeconds(); + case keyRBOMRQPercentage: + return provider.getBOMRQPercentage(); + case keyRBOMRequestAge: + return provider.getBOMRequestAge(); + case keyRBOMDuration: + return provider.getBOMDuration(); + case keyRDropOldRequestTime: + return provider.getRequestDropAgeHours(); + case keyRContentZoneInterval: + return provider.getContentZoneTimerInterval(); + case keyRJourneyTriggerEvents: + return provider.getJourneyTriggerEvents(); + case keyRJourneyTriggerViews: + return provider.getJourneyTriggerViews(); + default: + throw new IllegalArgumentException("no getter for settings key [" + key + "]"); + } + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java index 49197359..356f935d 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java @@ -57,11 +57,36 @@ static Config getBaseConfig() { } static Config getBaseConfig(String deviceID) { + Config config = getConfigSdkBehaviorSettings(deviceID); + // Disable server config requests by default in tests so the suite doesn't fire HTTP calls + // to a non-existent test host on every init. Tests covering ModuleConfiguration opt back in + // explicitly. + config.disableSdkBehaviorSettingsUpdates(); + return config; + } + + /** + * The base configuration with the SDK behavior settings requests left on, for the tests that + * exercise them. Pair it with an immediate request generator, or every init asks the test + * server for its settings. + * + * @return the configuration + */ + static Config getConfigSdkBehaviorSettings() { + return getConfigSdkBehaviorSettings(DEVICE_ID); + } + + /** + * The base configuration with the SDK behavior settings requests left on, for a given device ID. + * + * @param deviceID the custom device ID + * @return the configuration + */ + static Config getConfigSdkBehaviorSettings(String deviceID) { File sdkStorageRootDirectory = getTestSDirectory(); checkSdkStorageRootDirectoryExist(sdkStorageRootDirectory); Config config = new Config(SERVER_URL, SERVER_APP_KEY, sdkStorageRootDirectory); config.setApplicationVersion(APPLICATION_VERSION); - config.setCustomDeviceId(deviceID); return config; } @@ -698,4 +723,21 @@ static InternalConfig getConfigViews(Map segmentation) { config.views.setGlobalViewSegmentation(segmentation); return config; } + + /** + * Waits until the clock is past every timestamp handed out so far. A request file is named after + * a {@link TimeUtils#uniqueTimestampMs()} value, which is unique among the last ten values only, + * so requests stored back to back could reuse a name and overwrite each other. + */ + static void letTheClockCatchUp() { + long latest = TimeUtils.uniqueTimestampMs(); + while (System.currentTimeMillis() <= latest) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/UtilsInternalLimitsTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/UtilsInternalLimitsTests.java new file mode 100644 index 00000000..5c6e40b7 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/UtilsInternalLimitsTests.java @@ -0,0 +1,168 @@ +package ly.count.sdk.java.internal; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * The truncation rules of {@link UtilsInternalLimits} that the scenarios through the public API cannot + * reach on every platform: surrogate pairs, {@code \r\n} line breaks, {@code null} keys and the + * independence from the order of a map. + */ +@RunWith(JUnit4.class) +public class UtilsInternalLimitsTests { + + private static final String TAG = "[Caller] method"; + private final List warnings = new CopyOnWriteArrayList<>(); + private Log log; + + /** + * Starts every test with a logger that keeps every warning. + */ + @Before + public void beforeTest() { + warnings.clear(); + log = new Log(Config.LoggingLevel.OFF, (message, level) -> { + if (level == Config.LoggingLevel.WARN) { + warnings.add(message); + } + }); + } + + /** + * Keys and values are cut to the limit and never inside a surrogate pair. + *

+ * Verifies that {@code null}, empty and short strings come back as the same instance without a + * warning; that a longer string is cut to the limit with one warning naming the caller; that a + * cut falling inside a surrogate pair falls before it, or after it when the pair starts the + * string; and that only strings are cut by value, other types coming back as they are. + */ + @Test + public void strings_areCutToTheLimit_neverInsideASurrogatePair() { + String shortKey = "key"; + Assert.assertSame(shortKey, UtilsInternalLimits.truncateKey(shortKey, 3, log, TAG)); + Assert.assertNull(UtilsInternalLimits.truncateKey(null, 1, log, TAG)); + Assert.assertEquals("", UtilsInternalLimits.truncateValue("", 1, log, TAG)); + Assert.assertEquals(0, warnings.size()); + + Assert.assertEquals("abc", UtilsInternalLimits.truncateKey("abcdef", 3, log, TAG)); + Assert.assertEquals(1, warnings.size()); + Assert.assertTrue(warnings.get(0), warnings.get(0).startsWith(TAG + ", the key [abcdef] is longer than the key length limit of [3]")); + Assert.assertEquals("abcd", UtilsInternalLimits.truncateValue("abcdefgh", 4, log, TAG)); + Assert.assertTrue(warnings.get(1), warnings.get(1).startsWith(TAG + ", a value of [8] characters is longer than the value size limit of [4]")); + + String grinning = "😀"; + Assert.assertEquals("ab", UtilsInternalLimits.truncateValue("ab" + grinning + "cd", 3, log, TAG)); + Assert.assertEquals("ab" + grinning, UtilsInternalLimits.truncateValue("ab" + grinning + "cd", 4, log, TAG)); + Assert.assertEquals(grinning, UtilsInternalLimits.truncateKey(grinning + "rest", 1, log, TAG)); + + Integer number = 123456789; + Assert.assertSame(number, UtilsInternalLimits.truncateIfString(number, 2, log, TAG)); + Assert.assertEquals("12", UtilsInternalLimits.truncateIfString("123456789", 2, log, TAG)); + } + + /** + * A segmentation is cut and trimmed the same way whatever the order of its map, and never in place. + *

+ * Verifies that a segmentation within every limit, and {@code null}, come back as the same + * instance; that the same entries in two insertion orders give the same result, where of the keys + * cut to the same key the one that sorts last keeps its value and past the entry limit the keys + * that sort first are kept; that the caller's map is left as it was; that a {@code null} key + * sorts first and is never cut; and that cutting values only keeps every key in the order of the map. + */ + @Test + public void segmentation_isCutTheSameWayWhateverTheOrderOfTheMap_andNeverInPlace() { + Map within = TestUtils.map("abc", "value", "number", 12345678); + Assert.assertSame(within, UtilsInternalLimits.applySegmentationLimits(within, limits(6, 5, 2), log, TAG)); + Assert.assertNull(UtilsInternalLimits.applySegmentationLimits(null, limits(1, 1, 1), log, TAG)); + Assert.assertEquals(0, warnings.size()); + + Map forward = new LinkedHashMap<>(); + forward.put("colour_a", "red"); + forward.put("colour_b", "green"); + forward.put("colour_c", "blue"); + forward.put("amount", 5); + forward.put("zone", "europe_west"); + Map backward = new LinkedHashMap<>(); + List keys = Arrays.asList(forward.keySet().toArray(new String[0])); + Collections.reverse(keys); + for (String key : keys) { + backward.put(key, forward.get(key)); + } + Map forwardCopy = new HashMap<>(forward); + + Map expected = TestUtils.map("amoun", 5, "colou", "blue"); + Assert.assertEquals(expected, UtilsInternalLimits.applySegmentationLimits(forward, limits(5, 6, 2), log, TAG)); + Assert.assertEquals(expected, UtilsInternalLimits.applySegmentationLimits(backward, limits(5, 6, 2), log, TAG)); + Assert.assertEquals(Arrays.asList("amoun", "colou"), Arrays.asList(UtilsInternalLimits.applySegmentationLimits(backward, limits(5, 6, 2), log, TAG).keySet().toArray())); + Assert.assertEquals(forwardCopy, forward); + + Map withNullKey = new HashMap<>(); + withNullKey.put(null, "null_key_value"); + withNullKey.put("longer_key", 1); + Map limited = UtilsInternalLimits.applySegmentationLimits(withNullKey, limits(3, 100, 1), log, TAG); + Assert.assertEquals(Collections.singletonMap(null, "null_key_value"), limited); + + Map ordered = new LinkedHashMap<>(); + ordered.put("zeta", "long_value"); + ordered.put("alpha", 1); + ordered.put("mid", "short"); + Assert.assertSame(ordered, UtilsInternalLimits.truncateStringValues(ordered, 10, log, TAG)); + Map valuesCut = UtilsInternalLimits.truncateStringValues(ordered, 5, log, TAG); + Assert.assertEquals(Arrays.asList("zeta", "alpha", "mid"), Arrays.asList(valuesCut.keySet().toArray())); + Assert.assertEquals("long_", valuesCut.get("zeta")); + Assert.assertEquals("long_value", ordered.get("zeta")); + } + + /** + * Stack trace lines are cut without touching the line breaks, and the lines of a thread are + * counted from the top. + *

+ * Verifies that a stack trace within the limit comes back as the same instance; that {@code \n} + * and {@code \r\n} breaks, empty lines and a last line without a break are kept, the {@code \r} + * never counting as a character of its line; that one warning is logged per stack trace; and + * that the lines to keep of a thread are the limit only when the thread has more. + */ + @Test + public void stackTraces_linesAreCutWithoutTouchingTheLineBreaks_andLinesPerThreadCountFromTheTop() { + String within = "abc\r\nde\n\nf"; + Assert.assertSame(within, UtilsInternalLimits.truncateStackTraceLines(within, 3, log, TAG)); + + String stackTrace = "java.lang.Exception: message\r\n\tat first.frame(File.java:1)\n\n\tat last.frame(File.java:2)\r\nno_break_at_the_end"; + Assert.assertEquals("java.lang.\r\n\tat first.\n\n\tat last.f\r\nno_break_a", UtilsInternalLimits.truncateStackTraceLines(stackTrace, 10, log, TAG)); + Assert.assertEquals(1, warnings.size()); + Assert.assertTrue(warnings.get(0), warnings.get(0).startsWith(TAG + ", [4] stack trace lines are longer than the line length limit of [10]")); + Assert.assertEquals("12\r\n34\r\n", UtilsInternalLimits.truncateStackTraceLines("123\r\n345\r\n", 2, log, TAG)); + + Assert.assertEquals(5, UtilsInternalLimits.stackTraceLinesToKeep(5, 5, "worker", log, TAG)); + Assert.assertEquals(2, warnings.size()); + Assert.assertEquals(3, UtilsInternalLimits.stackTraceLinesToKeep(5, 3, "worker", log, TAG)); + Assert.assertTrue(warnings.get(2), warnings.get(2).startsWith(TAG + ", the stack trace of the thread [worker] has [5] lines, over the limit of [3]")); + } + + /** + * A settings provider serving the given key length, value size and segmentation entry limits. + * + * @param maxKeyLength the key length limit + * @param maxValueSize the value size limit + * @param maxSegmentationValues the segmentation entry limit + * @return the provider + */ + private static ConfigurationProvider limits(int maxKeyLength, int maxValueSize, int maxSegmentationValues) { + ModuleConfiguration limits = new ModuleConfiguration(); + limits.currentVMaxKeyLength = maxKeyLength; + limits.currentVMaxValueSize = maxValueSize; + limits.currentVMaxSegmentationValues = maxSegmentationValues; + return limits; + } +}