diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/pom.xml b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/pom.xml index 363fee6..b747f3c 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/pom.xml +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/pom.xml @@ -43,12 +43,6 @@ test-jar test - - eu.europa.ec.joinup.sat - jdplus-sts-base-core - 2.1.1-SNAPSHOT - test - eu.europa.ec.joinup.sat jdplus-tramoseats-base-core diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernel.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernel.java index 30039d2..3d4914b 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernel.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernel.java @@ -74,12 +74,8 @@ import jdplus.toolkit.base.core.regarima.RegArimaModel; import jdplus.toolkit.base.core.regarima.ami.GenericOutliersDetection; import jdplus.toolkit.base.core.regarima.ami.OutliersDetectionModule; -import jdplus.toolkit.base.core.ssf.arima.FastArimaForecasts; import jdplus.toolkit.base.core.ssf.arima.ExactArimaForecasts; -import jdplus.toolkit.base.core.ssf.arima.SsfUcarima; -import jdplus.toolkit.base.core.ssf.composite.CompositeSsf; import jdplus.toolkit.base.core.stats.likelihood.LogLikelihoodFunction; -import jdplus.toolkit.base.core.timeseries.simplets.Transformations; import org.checkerframework.checker.nullness.qual.NonNull; /** @@ -270,7 +266,7 @@ public static ExtendedAirlineEstimation fastProcess(DoubleSeq y, Matrix X, boole DataInterpolator interpolator = AverageInterpolator.interpolator(); double[] interpolatedData; - int[] missing = IntList.EMPTY; + int[] missing; if (y.anyMatch(z -> Double.isNaN(z))) { IntList lmissing = new IntList(); @@ -283,7 +279,6 @@ public static ExtendedAirlineEstimation fastProcess(DoubleSeq y, Matrix X, boole Arrays.sort(missing); } } else { - interpolatedData = null; missing = IntList.EMPTY; } @@ -309,10 +304,10 @@ public static ExtendedAirlineEstimation fastProcess(DoubleSeq y, Matrix X, boole .y(y) .addX(FastMatrix.of(X_withoutFcast)) .arima(mapping.getDefault()) - .meanCorrection(mean); - OutlierDescriptor[] o = null; - if (outliers != null && outliers.length - > 0) { + .meanCorrection(mean) + .missing(missing); + OutlierDescriptor[] o; + if (outliers != null && outliers.length > 0) { GlsArimaProcessor processor = GlsArimaProcessor.builder(ArimaModel.class) .precision(1e-5) .build(); @@ -358,7 +353,7 @@ public static ExtendedAirlineEstimation fastProcess(DoubleSeq y, Matrix X, boole //Ausgabe anpassen RegArimaModel model = rslt.getModel(); - DoubleSeq y_fcasts = DoubleSeq.empty(); + DoubleSeq y_fcasts; if (nfcasts > 0) { ExactArimaForecasts fcasts = new ExactArimaForecasts(); @@ -390,8 +385,8 @@ public static ExtendedAirlineEstimation fastProcess(DoubleSeq y, Matrix X, boole if (X != null && X.getColumnsCount() != 0) { for (int j = 0; j < X.getColumnsCount(); ++j) { double c = coeff.getAndNext(); - if (c != 0) { - for (int k = y.length(); k < y.length() + nfcasts; ++k) { + if (c != 0) { + for (int k = y.length(); k < y.length() + nfcasts; ++k) { y_fcast_a[k - y.length()] += c * X.column(j).get(k); } } @@ -470,7 +465,14 @@ private void execTransform(ExtendedRegAirlineModelling modelling, ProcessingLog log.push("log/level"); switch (spec.getTransform().getFunction()) { case Auto: - log.warning("not implemented yet. log used"); + LogLevelModule ll = LogLevelModule.builder() + .aiccLogCorrection(spec.getTransform().getAicDiff()) + .estimationPrecision(1e-5) + .build(); + + ll.process(modelling); + // log.warning("not implemented yet. log used"); +break; case Log: if (modelling.getDescription().getSeries().getValues().allMatch(x -> x > 0)) { modelling.getDescription().setLogTransformation(true); @@ -537,7 +539,7 @@ private void execOutliers(ExtendedRegAirlineModelling modelling, ProcessingLog l log.pop(); } - private static int outlierType(String[] all, String cur) { + public static int outlierType(String[] all, String cur) { for (int i = 0; i < all.length; ++i) { if (cur.equals(all[i])) { return i; @@ -546,7 +548,7 @@ private static int outlierType(String[] all, String cur) { return -1; } - private static IOutlierFactory[] factories(String[] code) { + public static IOutlierFactory[] factories(String[] code) { List fac = new ArrayList<>(); for (int i = 0; i < code.length; ++i) { switch (code[i]) { @@ -562,7 +564,7 @@ private static IOutlierFactory[] factories(String[] code) { return fac.toArray(IOutlierFactory[]::new); } - private static IOutlier outlier(String code, TsPeriod p) { + public static IOutlier outlier(String code, TsPeriod p) { LocalDateTime pos = p.start(); return switch (code) { case "ao", "AO" -> diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/LightExtendedAirlineEstimation.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/LightExtendedAirlineEstimation.java new file mode 100644 index 0000000..f1a6e8f --- /dev/null +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/LightExtendedAirlineEstimation.java @@ -0,0 +1,178 @@ +/* + * Copyright 2024 JDemetra+. + * Licensed under the EUPL, Version 1.2 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * https://joinup.ec.europa.eu/software/page/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package jdplus.highfreq.base.core.extendedairline; + +import java.util.List; +import jdplus.highfreq.base.api.ExtendedAirline; +import jdplus.highfreq.base.api.ExtendedAirlineSpec; +import jdplus.toolkit.base.api.data.DoubleSeq; +import jdplus.toolkit.base.api.data.DoubleSeqCursor; +import jdplus.toolkit.base.api.data.ParametersEstimation; +import jdplus.toolkit.base.api.information.GenericExplorable; +import jdplus.toolkit.base.api.math.matrices.Matrix; +import jdplus.toolkit.base.api.processing.ProcessingLog; +import jdplus.toolkit.base.api.timeseries.regression.MissingValueEstimation; +import jdplus.toolkit.base.core.arima.ArimaModel; +import jdplus.toolkit.base.core.data.DataBlock; +import jdplus.toolkit.base.core.regarima.RegArimaEstimation; +import jdplus.toolkit.base.core.regarima.RegArimaModel; +import jdplus.toolkit.base.core.regarima.RegArimaUtility; +import jdplus.toolkit.base.core.stats.likelihood.ConcentratedLikelihoodWithMissing; +import jdplus.toolkit.base.core.stats.likelihood.LikelihoodStatistics; +import jdplus.toolkit.base.core.stats.likelihood.LogLikelihoodFunction; + +/** + * + * @author Jean Palate + */ +@lombok.Value +@lombok.Builder(toBuilder = true, builderClassName = "Builder") +public class LightExtendedAirlineEstimation implements GenericExplorable { + + @lombok.NonNull + private DoubleSeq y; + @lombok.NonNull + private Matrix X; + private boolean mean; + + @lombok.NonNull + private ExtendedAirline model; + + /** + * Regression estimation. The order correspond to the order of the variables + * Fixed coefficients are not included + */ + @lombok.NonNull + private DoubleSeq coefficients; + @lombok.NonNull + private Matrix coefficientsCovariance; + + /** + * Positions corresponding to the estimation domain + */ + @lombok.NonNull + private MissingValueEstimation[] missing; + /** + * Parameters of the stochastic component. Fixed parameters are not included + */ + @lombok.NonNull + private ParametersEstimation parameters; + + private LikelihoodStatistics statistics; + + private DoubleSeq fullResiduals; + + @lombok.Singular + private List logs; + + public static Builder builder() { + Builder builder = new Builder(); + builder.y = DoubleSeq.empty(); + builder.X = Matrix.empty(); + builder.coefficients = DoubleSeq.empty(); + builder.coefficientsCovariance = Matrix.empty(); + builder.missing = NOMISSING; + builder.parameters = ParametersEstimation.empty(); + builder.fullResiduals = DoubleSeq.empty(); + return builder; + } + + private static final MissingValueEstimation[] NOMISSING = new MissingValueEstimation[0]; + + public static LightExtendedAirlineEstimation of(RegArimaEstimation estimation, ExtendedAirlineSpec spec) { + Builder builder = builder(); + RegArimaModel regarima = estimation.getModel(); + ConcentratedLikelihoodWithMissing ll = estimation.getConcentratedLikelihood(); + LogLikelihoodFunction.Point, ConcentratedLikelihoodWithMissing> max = estimation.getMax(); + int free = spec.freeParametersCount(); + int df = ll.degreesOfFreedom() - free; + double vscale = ll.ssq() / df; + builder.y(regarima.getY()) + .X(regarima.variables()) + .mean(regarima.isMean()) + .model(ExtendedAirline.of(spec.withFreeParameters(max.getParameters()))) + .coefficients(ll.coefficients()) + .coefficientsCovariance(ll.covariance(free, true)); + ParametersEstimation pestim = new ParametersEstimation(max.getParameters(), max.asymptoticCovariance(), max.getScore(), "Extended airline"); + builder.parameters(pestim) + .statistics(estimation.statistics()); + DoubleSeq fullRes = RegArimaUtility.fullResiduals(regarima, ll); + builder.fullResiduals(fullRes); + // complete for missings + int nmissing = ll.nmissing(); + if (nmissing > 0) { + DoubleSeq y = regarima.getY(); + MissingValueEstimation[] missing = new MissingValueEstimation[nmissing]; + DoubleSeqCursor cur = ll.missingCorrections().cursor(); + DoubleSeqCursor vcur = ll.missingUnscaledVariances().cursor(); + int[] pmissing = regarima.missing(); + for (int i = 0; i < nmissing; ++i) { + double m = cur.getAndNext(); + double v = vcur.getAndNext(); + missing[i] = new MissingValueEstimation(pmissing[i], y.get(pmissing[i]) - m, Math.sqrt(v * vscale)); + } + builder.missing(missing); + } + return builder.build(); + } + + public DoubleSeq originalY() { + if (y.anyMatch(z -> Double.isNaN(z))) { + // already contains the missing values + return y; + } + if (missing.length == 0) { + return y; + } + double[] z = y.toArray(); + for (int i = 0; i < missing.length; ++i) { + z[missing[i].getPosition()] = Double.NaN; + } + return DoubleSeq.of(z); + } + + public DoubleSeq linearizedData() { + double[] res = y.toArray(); + + // handle missing values + if (missing.length > 0) { + for (int i = 0; i < missing.length; ++i) { + res[missing[i].getPosition()] = missing[i].getValue(); + } + } + DataBlock e = DataBlock.of(res); + if (!X.isEmpty()) { + DoubleSeq regs = regressionEffect(); + e.sub(regs); + } + return e; + } + + public DoubleSeq regressionEffect() { + + if (coefficients.isEmpty()) { + return DoubleSeq.empty(); + } + + DataBlock d = DataBlock.make(y.length()); + d.setAY(coefficients.get(0), X.column(0)); + for (int i = 1; i < coefficients.length(); ++i) { + d.addAY(coefficients.get(i), X.column(i)); + } + return d; + } + +} diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/LogLevelModule.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/LogLevelModule.java new file mode 100644 index 0000000..e9b4565 --- /dev/null +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extendedairline/LogLevelModule.java @@ -0,0 +1,223 @@ +/* + * Copyright 2013 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package jdplus.highfreq.base.core.extendedairline; + +import jdplus.highfreq.base.core.regarima.ArimaComputer; +import jdplus.highfreq.base.core.regarima.ModelDescription; +import jdplus.toolkit.base.api.data.DoubleSeq; +import jdplus.toolkit.base.api.modelling.TransformationType; +import jdplus.toolkit.base.api.processing.ProcessingLog; +import jdplus.toolkit.base.core.arima.ArimaModel; +import jdplus.toolkit.base.core.data.transformation.LogJacobian; +import jdplus.toolkit.base.core.data.transformation.LogTransformation; +import jdplus.toolkit.base.core.regarima.RegArimaEstimation; +import jdplus.toolkit.base.core.regarima.RegArimaModel; +import jdplus.toolkit.base.core.regsarima.regular.ProcessingResult; +import jdplus.toolkit.base.core.sarima.SarimaModel; +import jdplus.toolkit.base.core.stats.likelihood.ConcentratedLikelihoodWithMissing; +import jdplus.toolkit.base.core.stats.likelihood.LogLikelihoodFunction; +import nbbrd.design.BuilderPattern; +import nbbrd.design.Development; + +/** + * Identification of log/level transformation + * + * @author Jean Palate + */ +@Development(status = Development.Status.Preliminary) +public class LogLevelModule { + + public static final String LL = "log-level test"; + + public static Builder builder() { + return new Builder(); + } + + @BuilderPattern(LogLevelModule.class) + public static class Builder { + + private double aiccdiff = -2; + private double precision = 1e-5; + + /** + * Precision used in the estimation of the models (1e-5 by default). In + * most cases, the precision can be smaller than for the estimation of + * the final model. + * + * @param eps + * @return + */ + public Builder estimationPrecision(double eps) { + this.precision = eps; + return this; + } + + /** + * Correction on the AICc of the model in logs. Negative corrections + * will favour logs (-2 by default) Same as aiccdiff in the original + * fortran program + * + * @param aiccdiff + * @return + */ + public Builder aiccLogCorrection(double aiccdiff) { + this.aiccdiff = aiccdiff; + return this; + } + + public LogLevelModule build() { + return new LogLevelModule(aiccdiff, precision); + } + + } + + private final double aiccDiff; + private final double precision; + private RegArimaEstimation level, log; + private double aiccLevel, aiccLog; + + private LogLevelModule(double aiccdiff, final double precision) { + this.aiccDiff = aiccdiff; + this.precision = precision; + } + + public double getEpsilon() { + return precision; + } + + /** + * + * @return + */ + public boolean isChoosingLog() { + if (log == null) { + return false; + } else if (level == null) { + return true; + } else { + // the best is the smallest (default aiccdiff is negative to favor logs) + return aiccLevel > aiccLog + aiccDiff; + } + } + + public boolean process(RegArimaModel regarima, ExtendedAirlineMapping mapping) { + clear(); + DoubleSeq y = regarima.getY(); + ArimaModel arima = regarima.arima(); + + if (y.anyMatch(z -> z <= 0)) { + return false; + } + ArimaComputer arimaComputer = new ArimaComputer(precision, false); + level = arimaComputer.process(regarima, mapping); + + int diff = arima.getNonStationaryArOrder(); + LogJacobian lj = new LogJacobian(diff, y.length(), regarima.missing()); + LogTransformation logs = new LogTransformation(); + DoubleSeq ly = logs.transform(y, lj); + RegArimaModel lregarima = regarima.toBuilder().y(ly).build(); + RegArimaEstimation rawLog = arimaComputer.process(lregarima, mapping); + if (rawLog != null) { + log = RegArimaEstimation.builder() + .model(rawLog.getModel()) + .concentratedLikelihood(rawLog.getConcentratedLikelihood()) + .max(rawLog.getMax()) + .llAdjustment(lj.value) + .build(); + + } + + if (level != null) { + aiccLevel = level.statistics().getAICC(); + } + if (log != null) { + aiccLog = log.statistics().getAICC(); + } + return true; + } + + public ProcessingResult process(ExtendedRegAirlineModelling modelling) { + clear(); + ProcessingLog logs = modelling.getLog(); + try { + logs.push(LL); + ModelDescription model = modelling.getDescription(); + if (model.getSeries().getValues().anyMatch(z -> z <= 0)) { + return ProcessingResult.Unchanged; + } + level = model.estimate(new ArimaComputer(precision, false)); + + ModelDescription logmodel = ModelDescription.copyOf(model); + logmodel.setLogTransformation(true); + log = logmodel.estimate(new ArimaComputer(precision, false)); + if (level != null) { + aiccLevel = level.statistics().getAICC(); + logs.info("level", level.statistics()); + } + if (log != null) { + aiccLog = log.statistics().getAICC(); + logs.info("Log", log.statistics()); + } + if (level == null && log == null) { + return ProcessingResult.Failed; + } + + if (isChoosingLog()) { + modelling.set(logmodel, log); + return ProcessingResult.Changed; + } else { + return ProcessingResult.Unchanged; + } + } finally { + logs.pop(); + } + } + + public TransformationType getTransformation() { + return this.isChoosingLog() ? TransformationType.Log : TransformationType.None; + } + + public double getAICcLevel() { + return this.aiccLevel; + } + + public double getAICcLog() { + return this.aiccLog; + } + + private void clear() { + log = null; + level = null; + aiccLevel = Double.NaN; + aiccLog = Double.NaN; + } + + /** + * @return the level_ + */ + public RegArimaEstimation getLevel() { + return level; + } + + /** + * @return the log_ + */ + public RegArimaEstimation getLog() { + return log; + } + +} diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/FractionalAirlineDecompositionExtractor.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineDecompositionExtractor.java similarity index 95% rename from jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/FractionalAirlineDecompositionExtractor.java rename to jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineDecompositionExtractor.java index deefdbe..73a3f15 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/FractionalAirlineDecompositionExtractor.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineDecompositionExtractor.java @@ -23,7 +23,7 @@ * @author palatej */ @ServiceProvider(InformationExtractor.class) -public class FractionalAirlineDecompositionExtractor extends InformationMapping { +public class ExtendedAirlineDecompositionExtractor extends InformationMapping { static final String Y = "y", T = "t", S = "s", I = "i", SA = "sa", T_E = "t_stde", S_E = "s_stde", I_E = "i_stde", NCMPS = "ncmps", CMP = "cmp", CMP_E = "cmp_stde", @@ -78,7 +78,7 @@ public double[] stde(LightExtendedAirlineDecomposition decomp, String cmp) { } } - public FractionalAirlineDecompositionExtractor() { + public ExtendedAirlineDecompositionExtractor() { delegate(LL, LikelihoodStatistics.class, r -> r.getLikelihood()); set(PCOV, Matrix.class, source -> source.getParametersCovariance()); set(PARAMETERS, double[].class, source -> source.getParameters().toArray()); diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/FractionalAirlineEstimationExtractor.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineEstimationExtractor.java similarity index 93% rename from jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/FractionalAirlineEstimationExtractor.java rename to jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineEstimationExtractor.java index 0bdbbbe..4805b07 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/FractionalAirlineEstimationExtractor.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineEstimationExtractor.java @@ -18,7 +18,7 @@ * @author palatej */ @ServiceProvider(InformationExtractor.class) -public class FractionalAirlineEstimationExtractor extends InformationMapping { +public class ExtendedAirlineEstimationExtractor extends InformationMapping { @Override public Class getSourceClass() { @@ -28,7 +28,7 @@ public Class getSourceClass() { private static final String PARAMETERS = "parameters", LL = "likelihood", RESIDUALS = "residuals", PCOV = "pcov", SCORE = "score", LOG = "log", MISSING = "missing", B = "b", T = "t", BVAR = "bvar", OUTLIERS = "outliers", LIN = "lin", REGRESSORS = "regressors", Y = "y", BNAMES = "variables", COMPONENT_AO = "component_ao", COMPONENT_LS = "component_ls", COMPONENT_WO = "component_wo", COMPONENT_OUTLIERS = "component_outliers", COMPONENT_USERDEF_REG_VARIABLES = "component_userdef_reg_variables"; - public FractionalAirlineEstimationExtractor() { + public ExtendedAirlineEstimationExtractor() { delegate(LL, LikelihoodStatistics.class, r -> r.getLikelihood()); set(LOG, Boolean.class, source -> source.isLog()); set(MISSING, int[].class, source -> source.getMissing()); diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractor.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractor.java index 1132323..a609858 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractor.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractor.java @@ -1,357 +1,48 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Copyright 2024 JDemetra+. + * Licensed under the EUPL, Version 1.2 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * https://joinup.ec.europa.eu/software/page/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. */ package jdplus.highfreq.base.core.extractors; -import jdplus.toolkit.base.api.data.DoubleSeq; -import jdplus.highfreq.base.api.ExtendedAirlineDictionaries; +import jdplus.highfreq.base.api.ExtendedAirline; +import jdplus.highfreq.base.core.extendedairline.LightExtendedAirlineEstimation; import jdplus.toolkit.base.api.information.InformationExtractor; import jdplus.toolkit.base.api.information.InformationMapping; -import jdplus.toolkit.base.api.modelling.ComponentInformation; -import jdplus.toolkit.base.api.modelling.SeriesInfo; -import jdplus.sa.base.api.ComponentType; -import jdplus.sa.base.api.DecompositionMode; -import jdplus.sa.base.api.SaDictionaries; -import jdplus.toolkit.base.api.timeseries.TsData; -import jdplus.toolkit.base.api.timeseries.TsDomain; -import jdplus.toolkit.base.api.dictionaries.Dictionary; -import jdplus.toolkit.base.api.dictionaries.RegressionDictionaries; -import java.util.Optional; -import jdplus.highfreq.base.core.extendedairline.decomposition.ExtendedAirlineDecomposition; -import jdplus.highfreq.base.core.extendedairline.ExtendedAirlineResults; -import jdplus.highfreq.base.core.regarima.HighFreqRegArimaModel; -import static jdplus.toolkit.base.core.regarima.extractors.RegSarimaModelExtractors.NBCAST; -import nbbrd.design.Development; import nbbrd.service.ServiceProvider; /** * - * @author PALATEJ + * @author Jean Palate */ -@Development(status = Development.Status.Release) @ServiceProvider(InformationExtractor.class) -public class ExtendedAirlineExtractor extends InformationMapping { - - public static final String FINAL = ""; +public class ExtendedAirlineExtractor extends InformationMapping { - private ExtendedAirlineDecomposition.Step step(ExtendedAirlineResults rslt, int period) { - Optional any = rslt.getDecomposition().getSteps().stream().filter(s -> period == (int) s.getPeriod()).findAny(); - return any.orElse(null); - } + public static String PERIODS = "periods", DIFF = "differencing", AR = "ar", + PHI = "phi", THETA = "theta", BTHETA = "btheta", PARAMETERS = "parameters"; - private String cmpItem(String key) { - return Dictionary.concatenate(SaDictionaries.DECOMPOSITION, key); + @Override + public Class getSourceClass() { + return ExtendedAirline.class; } public ExtendedAirlineExtractor() { - set(SaDictionaries.MODE, DecompositionMode.class, source -> source.getFinals().getMode()); - - set(RegressionDictionaries.Y, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Series, ComponentInformation.Value)); - set(SaDictionaries.T, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Trend, ComponentInformation.Value)); - set(SaDictionaries.SA, TsData.class, source - -> source.getFinals().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Value)); - set(SaDictionaries.S, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Seasonal, ComponentInformation.Value)); - set(SaDictionaries.I, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Irregular, ComponentInformation.Value)); - set(RegressionDictionaries.Y_F, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Series, ComponentInformation.Forecast)); - set(SaDictionaries.T+SaDictionaries.FORECAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Trend, ComponentInformation.Forecast)); - set(SaDictionaries.SA+SaDictionaries.FORECAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Forecast)); - set(SaDictionaries.S+SaDictionaries.FORECAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Seasonal, ComponentInformation.Forecast)); - set(SaDictionaries.I+SaDictionaries.FORECAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Irregular, ComponentInformation.Forecast)); - set(RegressionDictionaries.Y_B, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Series, ComponentInformation.Backcast)); - set(SaDictionaries.T+SaDictionaries.BACKCAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Trend, ComponentInformation.Backcast)); - set(SaDictionaries.SA+SaDictionaries.BACKCAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Backcast)); - set(SaDictionaries.S+SaDictionaries.BACKCAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Seasonal, ComponentInformation.Backcast)); - set(SaDictionaries.I+SaDictionaries.BACKCAST, TsData.class, source - -> source.getFinals().getSeries(ComponentType.Irregular, ComponentInformation.Backcast)); - set(cmpItem(SaDictionaries.T_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.Trend, ComponentInformation.Value)); - set(cmpItem(SaDictionaries.SA_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Value)); - set(cmpItem(SaDictionaries.S_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.Seasonal, ComponentInformation.Value)); - set(cmpItem(SaDictionaries.I_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.Irregular, ComponentInformation.Value)); - set(cmpItem(ExtendedAirlineDictionaries.T_CMP_F), TsData.class, source - -> source.getComponents().getSeries(ComponentType.Trend, ComponentInformation.Value)); - set(cmpItem(SaDictionaries.SA_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Value)); - set(cmpItem(SaDictionaries.S_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.Seasonal, ComponentInformation.Value)); - set(cmpItem(SaDictionaries.I_CMP), TsData.class, source - -> source.getComponents().getSeries(ComponentType.Irregular, ComponentInformation.Value)); - set(cmpItem(ExtendedAirlineDictionaries.SW_CMP), TsData.class, source -> { - DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SW_CMP); - if (data.isEmpty()) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_CMP_B), TsData.class, source -> { - int nb = source.getDecomposition().getBackcastsCount(); - if (nb == 0) { - return null; - } - DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SW_CMP); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_CMP_F), TsData.class, source -> { - int nf = source.getDecomposition().getForecastsCount(); - if (nf == 0) { - return null; - } - DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SW_CMP); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_CMP), TsData.class, source -> { - DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SY_CMP); - if (data.isEmpty()) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_CMP_B), TsData.class, source -> { - int nb = source.getDecomposition().getBackcastsCount(); - if (nb == 0) { - return null; - } - DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SY_CMP); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_CMP_F), TsData.class, source -> { - int nf = source.getDecomposition().getForecastsCount(); - if (nf == 0) { - return null; - } - DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SY_CMP); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_LIN), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 7); - if (step == null) { - return null; - } - DoubleSeq data = step.getComponent(1).getData(); - if (data.isEmpty()) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_B), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 7); - if (step == null) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(); - if (nb == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getData(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_F), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 7); - if (step == null) { - return null; - } - int nf = source.getDecomposition().getForecastsCount(); - if (nf == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getData(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_E), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 7); - if (step == null) { - return null; - } - DoubleSeq data = step.getComponent(1).getStde(); - if (data.isEmpty()) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_EB), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 7); - if (step == null) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(); - if (nb == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getStde(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_EF), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 7); - if (step == null) { - return null; - } - int nf = source.getDecomposition().getForecastsCount(); - if (nf == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getStde(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_LIN), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 365); - if (step == null) { - return null; - } - DoubleSeq data = step.getComponent(1).getData(); - if (data.isEmpty()) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_B), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 365); - if (step == null) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(); - if (nb == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getData(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_F), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 365); - if (step == null) { - return null; - } - int nf = source.getDecomposition().getForecastsCount(); - if (nf == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getData(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_E), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 365); - if (step == null) { - return null; - } - DoubleSeq data = step.getComponent(1).getStde(); - if (data.isEmpty()) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_EB), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 365); - if (step == null) { - return null; - } - int nb = source.getDecomposition().getBackcastsCount(); - if (nb == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getStde(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); - }); - set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_EF), TsData.class, source -> { - ExtendedAirlineDecomposition.Step step = step(source, 365); - if (step == null) { - return null; - } - int nf = source.getDecomposition().getForecastsCount(); - if (nf == 0) { - return null; - } - DoubleSeq data = step.getComponent(1).getStde(); - if (data.isEmpty()) { - return null; - } - TsDomain dom = source.getPreprocessing().getDescription().getDomain(); - return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); - }); - - delegate(null, HighFreqRegArimaModel.class, source -> source.getPreprocessing()); - - set(RegressionDictionaries.CAL, TsData.class, source -> source.getPreprocessing().getCalendarEffect(null)); - set(RegressionDictionaries.CAL + SeriesInfo.F_SUFFIX, TsData.class, - source -> source.getPreprocessing().getCalendarEffect(source.getPreprocessing().forecastDomain(source.getDecomposition().getForecastsCount()))); - setArray(RegressionDictionaries.CAL + SeriesInfo.B_SUFFIX, NBCAST, TsData.class, - (source, i) -> source.getPreprocessing().getCalendarEffect(source.getPreprocessing().backcastDomain(source.getDecomposition().getBackcastsCount()))); - } - - @Override - public Class getSourceClass() { - return ExtendedAirlineResults.class; + set(PARAMETERS, double[].class, source -> source.getP().toArray()); + set(PERIODS, double[].class, source -> source.getPeriodicities()); + set(DIFF, Integer.class, source -> source.getNdifferencing()); + set(AR, Boolean.class, source -> source.isAr()); + set(BTHETA, double[].class, source -> source.getP().drop(1, 0).toArray()); + set(PHI, Double.class, source -> source.isAr() ? source.getP().get(0) : Double.NaN); + set(THETA, Double.class, source -> source.isAr() ? Double.NaN : source.getP().get(0)); } } diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineResultsExtractor.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineResultsExtractor.java new file mode 100644 index 0000000..c2efa74 --- /dev/null +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineResultsExtractor.java @@ -0,0 +1,357 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package jdplus.highfreq.base.core.extractors; + +import jdplus.toolkit.base.api.data.DoubleSeq; +import jdplus.highfreq.base.api.ExtendedAirlineDictionaries; +import jdplus.toolkit.base.api.information.InformationExtractor; +import jdplus.toolkit.base.api.information.InformationMapping; +import jdplus.toolkit.base.api.modelling.ComponentInformation; +import jdplus.toolkit.base.api.modelling.SeriesInfo; +import jdplus.sa.base.api.ComponentType; +import jdplus.sa.base.api.DecompositionMode; +import jdplus.sa.base.api.SaDictionaries; +import jdplus.toolkit.base.api.timeseries.TsData; +import jdplus.toolkit.base.api.timeseries.TsDomain; +import jdplus.toolkit.base.api.dictionaries.Dictionary; +import jdplus.toolkit.base.api.dictionaries.RegressionDictionaries; +import java.util.Optional; +import jdplus.highfreq.base.core.extendedairline.decomposition.ExtendedAirlineDecomposition; +import jdplus.highfreq.base.core.extendedairline.ExtendedAirlineResults; +import jdplus.highfreq.base.core.regarima.HighFreqRegArimaModel; +import static jdplus.toolkit.base.core.regarima.extractors.RegSarimaModelExtractors.NBCAST; +import nbbrd.design.Development; +import nbbrd.service.ServiceProvider; + +/** + * + * @author PALATEJ + */ +@Development(status = Development.Status.Release) +@ServiceProvider(InformationExtractor.class) +public class ExtendedAirlineResultsExtractor extends InformationMapping { + + public static final String FINAL = ""; + + private ExtendedAirlineDecomposition.Step step(ExtendedAirlineResults rslt, int period) { + Optional any = rslt.getDecomposition().getSteps().stream().filter(s -> period == (int) s.getPeriod()).findAny(); + return any.orElse(null); + } + + private String cmpItem(String key) { + return Dictionary.concatenate(SaDictionaries.DECOMPOSITION, key); + } + + public ExtendedAirlineResultsExtractor() { + set(SaDictionaries.MODE, DecompositionMode.class, source -> source.getFinals().getMode()); + + set(RegressionDictionaries.Y, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Series, ComponentInformation.Value)); + set(SaDictionaries.T, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Trend, ComponentInformation.Value)); + set(SaDictionaries.SA, TsData.class, source + -> source.getFinals().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Value)); + set(SaDictionaries.S, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Seasonal, ComponentInformation.Value)); + set(SaDictionaries.I, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Irregular, ComponentInformation.Value)); + set(RegressionDictionaries.Y_F, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Series, ComponentInformation.Forecast)); + set(SaDictionaries.T+SaDictionaries.FORECAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Trend, ComponentInformation.Forecast)); + set(SaDictionaries.SA+SaDictionaries.FORECAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Forecast)); + set(SaDictionaries.S+SaDictionaries.FORECAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Seasonal, ComponentInformation.Forecast)); + set(SaDictionaries.I+SaDictionaries.FORECAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Irregular, ComponentInformation.Forecast)); + set(RegressionDictionaries.Y_B, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Series, ComponentInformation.Backcast)); + set(SaDictionaries.T+SaDictionaries.BACKCAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Trend, ComponentInformation.Backcast)); + set(SaDictionaries.SA+SaDictionaries.BACKCAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Backcast)); + set(SaDictionaries.S+SaDictionaries.BACKCAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Seasonal, ComponentInformation.Backcast)); + set(SaDictionaries.I+SaDictionaries.BACKCAST, TsData.class, source + -> source.getFinals().getSeries(ComponentType.Irregular, ComponentInformation.Backcast)); + set(cmpItem(SaDictionaries.T_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.Trend, ComponentInformation.Value)); + set(cmpItem(SaDictionaries.SA_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Value)); + set(cmpItem(SaDictionaries.S_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.Seasonal, ComponentInformation.Value)); + set(cmpItem(SaDictionaries.I_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.Irregular, ComponentInformation.Value)); + set(cmpItem(ExtendedAirlineDictionaries.T_CMP_F), TsData.class, source + -> source.getComponents().getSeries(ComponentType.Trend, ComponentInformation.Value)); + set(cmpItem(SaDictionaries.SA_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.SeasonallyAdjusted, ComponentInformation.Value)); + set(cmpItem(SaDictionaries.S_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.Seasonal, ComponentInformation.Value)); + set(cmpItem(SaDictionaries.I_CMP), TsData.class, source + -> source.getComponents().getSeries(ComponentType.Irregular, ComponentInformation.Value)); + set(cmpItem(ExtendedAirlineDictionaries.SW_CMP), TsData.class, source -> { + DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SW_CMP); + if (data.isEmpty()) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_CMP_B), TsData.class, source -> { + int nb = source.getDecomposition().getBackcastsCount(); + if (nb == 0) { + return null; + } + DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SW_CMP); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_CMP_F), TsData.class, source -> { + int nf = source.getDecomposition().getForecastsCount(); + if (nf == 0) { + return null; + } + DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SW_CMP); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_CMP), TsData.class, source -> { + DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SY_CMP); + if (data.isEmpty()) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_CMP_B), TsData.class, source -> { + int nb = source.getDecomposition().getBackcastsCount(); + if (nb == 0) { + return null; + } + DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SY_CMP); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_CMP_F), TsData.class, source -> { + int nf = source.getDecomposition().getForecastsCount(); + if (nf == 0) { + return null; + } + DoubleSeq data = source.getDecomposition().getFinalComponent(ExtendedAirlineDictionaries.SY_CMP); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_LIN), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 7); + if (step == null) { + return null; + } + DoubleSeq data = step.getComponent(1).getData(); + if (data.isEmpty()) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_B), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 7); + if (step == null) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(); + if (nb == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getData(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_F), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 7); + if (step == null) { + return null; + } + int nf = source.getDecomposition().getForecastsCount(); + if (nf == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getData(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_E), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 7); + if (step == null) { + return null; + } + DoubleSeq data = step.getComponent(1).getStde(); + if (data.isEmpty()) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_EB), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 7); + if (step == null) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(); + if (nb == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getStde(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SW_LIN_EF), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 7); + if (step == null) { + return null; + } + int nf = source.getDecomposition().getForecastsCount(); + if (nf == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getStde(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_LIN), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 365); + if (step == null) { + return null; + } + DoubleSeq data = step.getComponent(1).getData(); + if (data.isEmpty()) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_B), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 365); + if (step == null) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(); + if (nb == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getData(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_F), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 365); + if (step == null) { + return null; + } + int nf = source.getDecomposition().getForecastsCount(); + if (nf == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getData(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_E), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 365); + if (step == null) { + return null; + } + DoubleSeq data = step.getComponent(1).getStde(); + if (data.isEmpty()) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(), nf = source.getDecomposition().getForecastsCount(); + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod(), data.drop(nb, nf)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_EB), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 365); + if (step == null) { + return null; + } + int nb = source.getDecomposition().getBackcastsCount(); + if (nb == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getStde(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getStartPeriod().plus(-nb), data.range(0, nb)); + }); + set(cmpItem(ExtendedAirlineDictionaries.SY_LIN_EF), TsData.class, source -> { + ExtendedAirlineDecomposition.Step step = step(source, 365); + if (step == null) { + return null; + } + int nf = source.getDecomposition().getForecastsCount(); + if (nf == 0) { + return null; + } + DoubleSeq data = step.getComponent(1).getStde(); + if (data.isEmpty()) { + return null; + } + TsDomain dom = source.getPreprocessing().getDescription().getDomain(); + return TsData.of(dom.getEndPeriod(), data.range(data.length() - nf, data.length())); + }); + + delegate(null, HighFreqRegArimaModel.class, source -> source.getPreprocessing()); + + set(RegressionDictionaries.CAL, TsData.class, source -> source.getPreprocessing().getCalendarEffect(null)); + set(RegressionDictionaries.CAL + SeriesInfo.F_SUFFIX, TsData.class, + source -> source.getPreprocessing().getCalendarEffect(source.getPreprocessing().forecastDomain(source.getDecomposition().getForecastsCount()))); + setArray(RegressionDictionaries.CAL + SeriesInfo.B_SUFFIX, NBCAST, TsData.class, + (source, i) -> source.getPreprocessing().getCalendarEffect(source.getPreprocessing().backcastDomain(source.getDecomposition().getBackcastsCount()))); + } + + @Override + public Class getSourceClass() { + return ExtendedAirlineResults.class; + } +} diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/LightExtendedAirlineEstimationExtractor.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/LightExtendedAirlineEstimationExtractor.java new file mode 100644 index 0000000..04f0e19 --- /dev/null +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/jdplus/highfreq/base/core/extractors/LightExtendedAirlineEstimationExtractor.java @@ -0,0 +1,100 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package jdplus.highfreq.base.core.extractors; + +import jdplus.highfreq.base.api.ExtendedAirline; +import jdplus.highfreq.base.core.extendedairline.LightExtendedAirlineEstimation; +import jdplus.toolkit.base.api.information.InformationExtractor; +import jdplus.toolkit.base.api.information.InformationMapping; +import jdplus.toolkit.base.core.stats.likelihood.LikelihoodStatistics; +import jdplus.toolkit.base.api.math.matrices.Matrix; +import jdplus.toolkit.base.api.timeseries.regression.MissingValueEstimation; +import jdplus.toolkit.base.core.math.matrices.FastMatrix; +import nbbrd.service.ServiceProvider; + +/** + * + * @author palatej + */ +@ServiceProvider(InformationExtractor.class) +public class LightExtendedAirlineEstimationExtractor extends InformationMapping { + + @Override + public Class getSourceClass() { + return LightExtendedAirlineEstimation.class; + } + + private static final String PARAMETERS = "parameters", LL = "likelihood", MODEL = "model", RESIDUALS = "fullresiduals", PCOV = "pcov", SCORE = "score", MISSING = "missing", + B = "b", BVAR = "bvar", X = "X", Y = "y", MEAN = "mean", MISSING_POS = "missing_pos", MISSING_VAL = "missing_val", MISSING_STDEV = "missing_stdev", + LIN="linerized", REGEFFECT="regeffect"; + + public LightExtendedAirlineEstimationExtractor() { + set(B, double[].class, source -> source.getCoefficients().toArray()); + set(BVAR, Matrix.class, source -> source.getCoefficientsCovariance()); + set(Y, double[].class, source -> source.getY().toArray()); + set(X, double[].class, source -> source.getY().toArray()); + set(MEAN, Boolean.class, source -> source.isMean()); + delegate(MODEL, ExtendedAirline.class, r -> r.getModel()); + set(PARAMETERS, double[].class, source -> source.getParameters().getValues().toArray()); + set(SCORE, double[].class, source -> source.getParameters().getScores().toArray()); + set(PCOV, Matrix.class, source -> source.getParameters().getCovariance()); + set(RESIDUALS, double[].class, source -> source.getFullResiduals().toArray()); + set(MISSING, Matrix.class, source -> { + MissingValueEstimation[] missing = source.getMissing(); + if (missing.length > 0) { + FastMatrix M=FastMatrix.make(missing.length, 3); + for (int i = 0; i < missing.length; ++i) { + M.set(i, 0, missing[i].getPosition()); + M.set(i, 1, missing[i].getValue()); + M.set(i, 2, missing[i].getStandardError()); + } + return M; + } else { + return null; + } + }); + set(MISSING_POS, int[].class, source -> { + MissingValueEstimation[] missing = source.getMissing(); + if (missing.length > 0) { + int[] pos = new int[missing.length]; + for (int i = 0; i < pos.length; ++i) { + pos[i] = missing[i].getPosition(); + } + return pos; + } else { + return null; + } + }); + set(MISSING_VAL, double[].class, source -> { + MissingValueEstimation[] missing = source.getMissing(); + if (missing.length > 0) { + double[] val = new double[missing.length]; + for (int i = 0; i < val.length; ++i) { + val[i] = missing[i].getValue(); + } + return val; + } else { + return null; + } + }); + set(MISSING_STDEV, double[].class, source -> { + MissingValueEstimation[] missing = source.getMissing(); + if (missing.length > 0) { + double[] e = new double[missing.length]; + for (int i = 0; i < e.length; ++i) { + e[i] = missing[i].getStandardError(); + } + return e; + } else { + return null; + } + }); + delegate(LL, LikelihoodStatistics.class, r -> r.getStatistics()); + set(LIN, double[].class, source -> source.linearizedData().toArray()); + set(REGEFFECT, double[].class, source -> source.regressionEffect().toArray()); + + } +} diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/module-info.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/module-info.java index bc465d2..199523b 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/module-info.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/main/java/module-info.java @@ -1,7 +1,9 @@ +import jdplus.highfreq.base.core.extractors.ExtendedAirlineResultsExtractor; +import jdplus.highfreq.base.core.extractors.ExtendedAirlineDecompositionExtractor; +import jdplus.highfreq.base.core.extractors.ExtendedAirlineEstimationExtractor; import jdplus.highfreq.base.core.extractors.ExtendedAirlineExtractor; -import jdplus.highfreq.base.core.extractors.FractionalAirlineDecompositionExtractor; -import jdplus.highfreq.base.core.extractors.FractionalAirlineEstimationExtractor; import jdplus.highfreq.base.core.extractors.HighFreqRegArimaExtractor; +import jdplus.highfreq.base.core.extractors.LightExtendedAirlineEstimationExtractor; import jdplus.toolkit.base.api.information.InformationExtractor; module jdplus.highfreq.base.core { @@ -25,7 +27,9 @@ provides InformationExtractor with ExtendedAirlineExtractor, - FractionalAirlineDecompositionExtractor, + ExtendedAirlineResultsExtractor, + ExtendedAirlineDecompositionExtractor, HighFreqRegArimaExtractor, - FractionalAirlineEstimationExtractor; + ExtendedAirlineEstimationExtractor, + LightExtendedAirlineEstimationExtractor; } \ No newline at end of file diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernelTest.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernelTest.java index cdae3b1..97006f3 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernelTest.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/ExtendedAirlineKernelTest.java @@ -57,7 +57,7 @@ public void testDaily() throws IOException { // build the psec ExtendedAirlineModellingSpec spec=ExtendedAirlineModellingSpec.builder() .transform(TransformSpec.builder() - .function(TransformationType.Log) + .function(TransformationType.Auto) .build()) .stochastic(ExtendedAirlineSpec.DEFAULT_WD) .outlier(OutlierSpec.builder() diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/MultiPeriodicAirlineMappingTest.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/MultiPeriodicAirlineMappingTest.java index a09b49f..fd68c4e 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/MultiPeriodicAirlineMappingTest.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extendedairline/MultiPeriodicAirlineMappingTest.java @@ -44,11 +44,11 @@ import jdplus.toolkit.base.core.data.DataBlockStorage; import jdplus.toolkit.base.core.math.linearfilters.RationalFilter; import jdplus.toolkit.base.core.math.matrices.FastMatrix; -import jdplus.sts.base.core.msts.AtomicModels; -import jdplus.sts.base.core.msts.CompositeModel; -import jdplus.sts.base.core.msts.CompositeModelEstimation; -import jdplus.sts.base.core.msts.ModelEquation; -import jdplus.sts.base.core.msts.StateItem; +//import jdplus.sts.base.core.msts.AtomicModels; +//import jdplus.sts.base.core.msts.CompositeModel; +//import jdplus.sts.base.core.msts.CompositeModelEstimation; +//import jdplus.sts.base.core.msts.ModelEquation; +//import jdplus.sts.base.core.msts.StateItem; import jdplus.toolkit.base.core.ssf.dk.DkToolkit; import jdplus.toolkit.base.core.ssf.composite.CompositeSsf; import jdplus.toolkit.base.core.ssf.composite.MultivariateCompositeSsf; @@ -226,36 +226,36 @@ public static void testDaily3() throws IOException { } // @Test - public static void testDailySts() throws IOException { - InputStream stream = Data.class.getResourceAsStream("edf.txt"); - Matrix edf = MatrixSerializer.read(stream); - CompositeModel model = new CompositeModel(); - StateItem l = AtomicModels.localLinearTrend("l", .01, 0.01, false, false); - StateItem sd = AtomicModels.seasonalComponent("sd", "HarrisonStevens", 7, .01, false); - StateItem sy = AtomicModels.seasonalComponent("sy", "HarrisonStevens", 365, .01, false); - StateItem n = AtomicModels.noise("n", .01, false); - ModelEquation eq = new ModelEquation("eq1", 0, true); - eq.add(l); - eq.add(sd); - eq.add(sy); - eq.add(n); - model.add(l); - model.add(sd); - model.add(sy); - model.add(n); - model.add(eq); - FastMatrix M = FastMatrix.of(edf); - M.apply(Math::log); - CompositeModelEstimation rslt = model.estimate(M, false, true, SsfInitialization.Diffuse, Optimizer.LevenbergMarquardt, 1e-6, null); - MultivariateCompositeSsf ssf = rslt.getSsf(); - ISsf ussf = M2uAdapter.of(ssf); - ISsfData udata = new SsfData(M.column(0)); - DataBlockStorage ds = DkToolkit.fastSmooth(ussf, udata); - int[] cmpPos = rslt.getCmpPos(); - for (int i = 0; i < cmpPos.length; ++i) { - System.out.println(ds.item(cmpPos[i])); - } - } +// public static void testDailySts() throws IOException { +// InputStream stream = Data.class.getResourceAsStream("edf.txt"); +// Matrix edf = MatrixSerializer.read(stream); +// CompositeModel model = new CompositeModel(); +// StateItem l = AtomicModels.localLinearTrend("l", .01, 0.01, false, false); +// StateItem sd = AtomicModels.seasonalComponent("sd", "HarrisonStevens", 7, .01, false); +// StateItem sy = AtomicModels.seasonalComponent("sy", "HarrisonStevens", 365, .01, false); +// StateItem n = AtomicModels.noise("n", .01, false); +// ModelEquation eq = new ModelEquation("eq1", 0, true); +// eq.add(l); +// eq.add(sd); +// eq.add(sy); +// eq.add(n); +// model.add(l); +// model.add(sd); +// model.add(sy); +// model.add(n); +// model.add(eq); +// FastMatrix M = FastMatrix.of(edf); +// M.apply(Math::log); +// CompositeModelEstimation rslt = model.estimate(M, false, true, SsfInitialization.Diffuse, Optimizer.LevenbergMarquardt, 1e-6, null); +// MultivariateCompositeSsf ssf = rslt.getSsf(); +// ISsf ussf = M2uAdapter.of(ssf); +// ISsfData udata = new SsfData(M.column(0)); +// DataBlockStorage ds = DkToolkit.fastSmooth(ussf, udata); +// int[] cmpPos = rslt.getCmpPos(); +// for (int i = 0; i < cmpPos.length; ++i) { +// System.out.println(ds.item(cmpPos[i])); +// } +// } @Test @Disabled diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractorTest.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractorTest.java index 942cfb5..3bb83d0 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractorTest.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-core/src/test/java/jdplus/highfreq/base/core/extractors/ExtendedAirlineExtractorTest.java @@ -18,6 +18,7 @@ import java.util.LinkedHashMap; import jdplus.highfreq.base.core.extendedairline.ExtendedAirlineEstimation; +import jdplus.highfreq.base.core.extendedairline.LightExtendedAirlineEstimation; import jdplus.toolkit.base.api.information.InformationExtractors; import org.junit.jupiter.api.Test; @@ -38,6 +39,14 @@ public static void main(String[] args) { for (String s : dic.keySet()){ System.out.println(s); } + System.out.println(""); + dic=new LinkedHashMap<>(); + InformationExtractors.fillDictionary(LightExtendedAirlineEstimation.class, null, dic, true); + + for (String s : dic.keySet()){ + System.out.println(s); + } + } } diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/pom.xml b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/pom.xml index d391ca0..496a866 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/pom.xml +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/pom.xml @@ -41,12 +41,12 @@ ${jd2.version} test - + eu.europa.ec.joinup.sat jdplus-toolkit-base-api diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/jdplus/highfreq/base/r/ExtendedAirlineProcessor.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/jdplus/highfreq/base/r/ExtendedAirlineProcessor.java new file mode 100644 index 0000000..1b8f113 --- /dev/null +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/jdplus/highfreq/base/r/ExtendedAirlineProcessor.java @@ -0,0 +1,150 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package jdplus.highfreq.base.r; + +import java.util.Arrays; +import static jdplus.sa.base.core.PreliminaryChecks.MAX_MISSING_COUNT; +import jdplus.highfreq.base.api.ExtendedAirlineSpec; +import static jdplus.highfreq.base.core.extendedairline.ExtendedAirlineKernel.factories; +import jdplus.highfreq.base.core.extendedairline.ExtendedAirlineMapping; +import jdplus.highfreq.base.core.extendedairline.LightExtendedAirlineEstimation; +import jdplus.highfreq.base.core.extendedairline.LogLevelModule; +import jdplus.sa.base.api.SaException; +import jdplus.toolkit.base.api.data.DoubleSeq; +import jdplus.toolkit.base.api.data.Parameter; +import jdplus.toolkit.base.api.math.matrices.Matrix; +import jdplus.toolkit.base.api.util.IntList; +import jdplus.toolkit.base.core.arima.ArimaModel; +import jdplus.toolkit.base.core.data.interpolation.AverageInterpolator; +import jdplus.toolkit.base.core.data.interpolation.DataInterpolator; +import jdplus.toolkit.base.core.math.functions.levmar.LevenbergMarquardtMinimizer; +import jdplus.toolkit.base.core.math.matrices.FastMatrix; +import jdplus.toolkit.base.core.modelling.regression.IOutlierFactory; +import jdplus.toolkit.base.core.regarima.GlsArimaProcessor; +import jdplus.toolkit.base.core.regarima.RegArimaEstimation; +import jdplus.toolkit.base.core.regarima.RegArimaModel; +import jdplus.toolkit.base.core.regarima.ami.GenericOutliersDetection; +import jdplus.toolkit.base.core.regarima.ami.OutliersDetectionModule; + +/** + * Upgrade to highfreqregarima + * + * + * @author Jean Palate + */ +@lombok.experimental.UtilityClass +public class ExtendedAirlineProcessor { + + public ExtendedAirlineSpec spec(double[] periods, int ndiff, boolean ar, boolean toInt) { + return ExtendedAirlineSpec.builder() + .periodicities(periods) + .differencingOrder(ndiff) + .phi(ar ? Parameter.undefined() : null) + .theta(ar ? null : Parameter.undefined()) + .adjustToInt(toInt) + .build(); + } + + public RegArimaModel regarima(double[] Y, boolean mean, Matrix X, ExtendedAirlineSpec spec) { + DoubleSeq y = DoubleSeq.of(Y); + //Missing + int nz = y.length(); + int nm = y.count(z -> !Double.isFinite(z)); + if (nm > MAX_MISSING_COUNT * nz / 100) { + throw new SaException("Too many missing values"); + } + + DataInterpolator interpolator = AverageInterpolator.interpolator(); + double[] interpolatedData; + int[] missing; + + if (y.anyMatch(z -> Double.isNaN(z))) { + IntList lmissing = new IntList(); + interpolatedData = interpolator.interpolate(y, lmissing); + y = DoubleSeq.of(interpolatedData); + if (lmissing.isEmpty()) { + missing = IntList.EMPTY; + } else { + missing = lmissing.toArray(); + Arrays.sort(missing); + } + } else { + missing = IntList.EMPTY; + } + + final ExtendedAirlineMapping mapping = ExtendedAirlineMapping.of(spec); + // + return RegArimaModel.builder() + .y(y) + .addX(FastMatrix.of(X)) + .arima(mapping.getDefault()) + .missing(missing) + .meanCorrection(mean) + .build(); + } + + public LightExtendedAirlineEstimation estimate(RegArimaModel regarima, ExtendedAirlineSpec spec, double eps, boolean exactderivatives) { + final ExtendedAirlineMapping mapping = ExtendedAirlineMapping.of(spec); + GlsArimaProcessor processor = GlsArimaProcessor.builder(ArimaModel.class) + .precision(eps) + .computeExactFinalDerivatives(exactderivatives) + .build(); + RegArimaEstimation rslt = processor.process(regarima, mapping); + return LightExtendedAirlineEstimation.of(rslt, spec); + } + + public double[] logLevelTest(RegArimaModel model, ExtendedAirlineSpec spec, double eps) { + LogLevelModule ll = LogLevelModule + .builder() + .estimationPrecision(eps) + .build(); + if (ll.process(model, ExtendedAirlineMapping.of(spec))) { + return new double[]{ll.getAICcLevel(), ll.getAICcLog()}; + } else { + return null; + } + } + + public Matrix outliers(RegArimaModel regarima, ExtendedAirlineSpec spec, String[] outliers, int start, int end, double cv, int maxoutliers, int maxround) { + LevenbergMarquardtMinimizer.LmBuilder min = LevenbergMarquardtMinimizer.builder().maxIter(5); + GlsArimaProcessor processor = GlsArimaProcessor.builder(ArimaModel.class + ) + .minimizer(min) + .precision(1e-5) + .build(); + IOutlierFactory[] factories = factories(outliers); + OutliersDetectionModule od = OutliersDetectionModule.build(ArimaModel.class + ) + .maxOutliers(maxoutliers) + .maxRound(maxround) + .addFactories(factories) + .processor(processor) + .build(); + + DoubleSeq y = regarima.getY(); + cv = Math.max(cv, GenericOutliersDetection.criticalValue(y.length(), 0.01)); + od.setCriticalValue(cv); + od.prepare(y.length()); + od.setBounds(start, end == 0 ? y.length() : end); + // remove missing values + int[] missing = regarima.missing(); + for (int i = 0; i < missing.length; ++i) { + for (int j = 0; j < outliers.length; ++j) { + od.exclude(missing[i], j); + } + } + + od.process(regarima, ExtendedAirlineMapping.of(spec)); + int[][] io = od.getOutliers(); + if (io.length > 0) { + FastMatrix M = FastMatrix.make(io.length, 2); + M.set((i, j) -> io[i][j]); + return M; + } else { + return null; + } + } + +} diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/module-info.java b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/module-info.java index 250c723..b0f8de5 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/module-info.java +++ b/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/main/java/module-info.java @@ -9,6 +9,7 @@ requires jdplus.toolkit.base.api; requires jdplus.highfreq.base.core; requires jdplus.toolkit.base.core; + requires jdplus.sa.base.core; exports jdplus.highfreq.base.r; } \ No newline at end of file diff --git a/jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-api/src/main/java/jdplus/sts/base/api/StsSpec.java b/jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-api/src/main/java/jdplus/sts/base/api/StsSpec.java index b9efe3b..50b6245 100644 --- a/jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-api/src/main/java/jdplus/sts/base/api/StsSpec.java +++ b/jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-api/src/main/java/jdplus/sts/base/api/StsSpec.java @@ -11,7 +11,7 @@ @lombok.Value @lombok.Builder(toBuilder = true, builderClassName = "Builder") public class StsSpec implements SaSpecification{ - public static final String METHOD = "stlplus"; + public static final String METHOD = "sts"; public static final String VERSION_V3 = "3.0.0"; public static final AlgorithmDescriptor DESCRIPTOR = new AlgorithmDescriptor(FAMILY, METHOD, VERSION_V3); diff --git a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/test/java/jdplus/highfreq/base/r/SplinesTest.java b/jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-r/src/test/java/jdplus/sts/base/r/SplinesTest.java similarity index 98% rename from jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/test/java/jdplus/highfreq/base/r/SplinesTest.java rename to jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-r/src/test/java/jdplus/sts/base/r/SplinesTest.java index 8bf1790..05dad9a 100644 --- a/jdplus-incubator-base/jdplus-highfreq-base-parent/jdplus-highfreq-base-r/src/test/java/jdplus/highfreq/base/r/SplinesTest.java +++ b/jdplus-incubator-base/jdplus-sts-base-parent/jdplus-sts-base-r/src/test/java/jdplus/sts/base/r/SplinesTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package jdplus.highfreq.base.r; +package jdplus.sts.base.r; import jdplus.toolkit.base.api.data.DoubleSeq; import tck.demetra.data.MatrixSerializer; @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import jdplus.highfreq.base.core.extendedairline.ExtendedAirlineMapping; import jdplus.toolkit.base.core.math.matrices.FastMatrix; import jdplus.sts.base.core.msts.AtomicModels; import jdplus.sts.base.core.msts.CompositeModel; diff --git a/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/StsViewFactory.java b/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/StsViewFactory.java index 321bdf8..ea17f32 100644 --- a/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/StsViewFactory.java +++ b/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/StsViewFactory.java @@ -67,10 +67,10 @@ */ public class StsViewFactory extends ProcDocumentViewFactory { - public static final String SELECTION = "Selection", + public static final String SELECTION = "STS estimation", STOCHASTIC = "Stochastic series", COMPONENTS = "Components", - STM = "Sts components", + STM = "STS components", MODELBASED = "Model-based tests", WKANALYSIS = "WK analysis", WK_COMPONENTS = "Components", diff --git a/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/actions/OpenStlDoc.java b/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/actions/OpenStsDoc.java similarity index 93% rename from jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/actions/OpenStlDoc.java rename to jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/actions/OpenStsDoc.java index 8c91ea1..44f2d3f 100644 --- a/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/actions/OpenStlDoc.java +++ b/jdplus-incubator-desktop/jdplus-sts-desktop-plugin/src/main/java/jdplus/sts/desktop/plugin/bsm/ui/actions/OpenStsDoc.java @@ -24,11 +24,11 @@ @ActionReference(path = StsDocumentManager.ITEMPATH, position = 1600, separatorBefore = 1590) }) @NbBundle.Messages("CTL_OpenStsDoc=Open") -public class OpenStlDoc implements ActionListener { +public class OpenStsDoc implements ActionListener { private final WsNode context; - public OpenStlDoc(WsNode context) { + public OpenStsDoc(WsNode context) { this.context = context; }