diff --git a/tools/product_config/src/com/android/build/config/ConfigBase.java b/tools/product_config/src/com/android/build/config/ConfigBase.java index 5ac1fc2bf..0c67d1686 100644 --- a/tools/product_config/src/com/android/build/config/ConfigBase.java +++ b/tools/product_config/src/com/android/build/config/ConfigBase.java @@ -63,6 +63,10 @@ public class ConfigBase { mProductVars.put(name, type); } + public TreeMap getProductVars() { + return mProductVars; + } + public VarType getVarType(String name) { final VarType t = mProductVars.get(name); if (t != null) { @@ -75,4 +79,15 @@ public class ConfigBase { public boolean isProductVar(String name) { return mProductVars.get(name) != null; } + + /** + * Copy common base class fields from that to this. + */ + public void copyFrom(ConfigBase that) { + setPhase(that.getPhase()); + setRootNodes(that.getRootNodes()); + for (Map.Entry entry: that.getProductVars().entrySet()) { + addProductVar(entry.getKey(), entry.getValue()); + } + } } diff --git a/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java b/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java new file mode 100644 index 000000000..369d4d62c --- /dev/null +++ b/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Converts a MakeConfig into a Generic config by applying heuristics about + * the types of variable assignments that we do. + */ +public class ConvertMakeToGenericConfig { + private final Errors mErrors; + + public ConvertMakeToGenericConfig(Errors errors) { + mErrors = errors; + } + + public GenericConfig convert(MakeConfig make) { + final GenericConfig result = new GenericConfig(); + + // Base class fields + result.copyFrom(make); + + // Each file + for (MakeConfig.ConfigFile f: make.getConfigFiles()) { + final GenericConfig.ConfigFile genericFile + = new GenericConfig.ConfigFile(f.getFilename()); + result.addConfigFile(genericFile); + + final List blocks = f.getBlocks(); + + // Some assertions: + // TODO: Include better context for these errors. + // There should always be at least a BEGIN and an AFTER, so assert this. + if (blocks.size() < 2) { + throw new RuntimeException("expected at least blocks.size() >= 2. Actcual size: " + + blocks.size()); + } + if (blocks.get(0).getBlockType() != MakeConfig.BlockType.BEFORE) { + throw new RuntimeException("expected first block to be BEFORE"); + } + if (blocks.get(blocks.size() - 1).getBlockType() != MakeConfig.BlockType.AFTER) { + throw new RuntimeException("expected first block to be AFTER"); + } + // Everything in between should be an INHERIT block. + for (int index = 1; index < blocks.size() - 1; index++) { + if (blocks.get(index).getBlockType() != MakeConfig.BlockType.INHERIT) { + throw new RuntimeException("expected INHERIT at block " + index); + } + } + + // Each block represents a snapshot of the interpreter variable state (minus a few big + // sets of variables which we don't export because they're used in the internals + // of node_fns.mk, so we know they're not necessary here). The first (BEFORE) one + // is everything that is set before the file is included, so it forms the base + // for everything else. + MakeConfig.Block prevBlock = blocks.get(0); + + for (int index = 1; index < blocks.size(); index++) { + final MakeConfig.Block block = blocks.get(index); + for (final Map.Entry entry: block.getVars().entrySet()) { + final String varName = entry.getKey(); + final GenericConfig.Assign assign = convertAssignment(block.getBlockType(), + block.getInheritedFile(), make.getVarType(varName), varName, + entry.getValue(), prevBlock.getVar(varName)); + if (assign != null) { + genericFile.addStatement(assign); + } + } + // Handle variables that are in prevBlock but not block -- they were + // deleted. Is this even possible, or do they show up as ""? We will + // treat them as positive assigments to empty string + for (String prevName: prevBlock.getVars().keySet()) { + if (!block.getVars().containsKey(prevName)) { + genericFile.addStatement( + new GenericConfig.Assign(prevName, new Str(""))); + } + } + if (block.getBlockType() == MakeConfig.BlockType.INHERIT) { + genericFile.addStatement( + new GenericConfig.Inherit(block.getInheritedFile())); + } + // For next iteration + prevBlock = block; + } + } + return result; + } + + /** + * Converts one variable from a MakeConfig Block into a GenericConfig Assignment. + */ + GenericConfig.Assign convertAssignment(MakeConfig.BlockType blockType, Str inheritedFile, + ConfigBase.VarType varType, String varName, Str varVal, Str prevVal) { + if (prevVal == null) { + // New variable. + return new GenericConfig.Assign(varName, varVal); + } else if (!varVal.equals(prevVal)) { + // The value changed from the last block. + if (varVal.equals("")) { + // It was set to empty + return new GenericConfig.Assign(varName, varVal); + } else { + // Product vars have the @inherit processing. Other vars we + // will just ignore and put in one section at the end, based + // on the difference between the BEFORE and AFTER blocks. + if (varType == ConfigBase.VarType.UNKNOWN) { + if (blockType == MakeConfig.BlockType.AFTER) { + // For UNKNOWN variables, we don't worry about the + // intermediate steps, just take the final value. + return new GenericConfig.Assign(varName, varVal); + } else { + return null; + } + } else { + return convertInheritedVar(blockType, inheritedFile, + varName, varVal, prevVal); + } + } + } else { + // Variable not touched + return null; + } + } + + /** + * Handle the special inherited values, where the inherit-product puts in the + * @inherit:... markers, adding Statements to the ConfigFile. + */ + GenericConfig.Assign convertInheritedVar(MakeConfig.BlockType blockType, Str inheritedFile, + String varName, Str varVal, Str prevVal) { + String varText = varVal.toString(); + String prevText = prevVal.toString().trim(); + if (blockType == MakeConfig.BlockType.INHERIT) { + // inherit-product appends @inherit:... so drop that. + final String marker = "@inherit:" + inheritedFile; + if (varText.endsWith(marker)) { + varText = varText.substring(0, varText.length() - marker.length()).trim(); + } else { + mErrors.ERROR_IMPROPER_PRODUCT_VAR_MARKER.add(varVal.getPosition(), + "Variable didn't end with marker \"" + marker + "\": " + varText); + } + } + + if (!varText.equals(prevText)) { + // If the variable value was actually changed. + final ArrayList words = split(varText, prevText); + if (words.size() == 0) { + // Pure Assignment, none of the previous value is present. + return new GenericConfig.Assign(varName, new Str(varVal.getPosition(), varText)); + } else { + // Self referential value (prepend, append, both). + if (words.size() > 2) { + // This is indicative of a construction that might not be quite + // what we want. The above code will do something that works if it was + // of the form "VAR := a $(VAR) b $(VAR) c", but if the original code + // something else this won't work. This doesn't happen in AOSP, but + // it's a theoretically possibility, so someone might do it. + mErrors.WARNING_VARIABLE_RECURSION.add(varVal.getPosition(), + "Possible unsupported variable recursion: " + + varName + " = " + varVal + " (prev=" + prevVal + ")"); + } + return new GenericConfig.Assign(varName, Str.toList(varVal.getPosition(), words)); + } + } else { + // Variable not touched + return null; + } + } + + /** + * Split 'haystack' on occurrences of 'needle'. Trims each string of whitespace + * to preserve make list semantics. + */ + private static ArrayList split(String haystack, String needle) { + final ArrayList result = new ArrayList(); + final int needleLen = needle.length(); + if (needleLen == 0) { + return result; + } + int start = 0; + int end; + while ((end = haystack.indexOf(needle, start)) >= 0) { + result.add(haystack.substring(start, end).trim()); + start = end + needleLen; + } + result.add(haystack.substring(start).trim()); + return result; + } +} diff --git a/tools/product_config/src/com/android/build/config/DumpConfigParser.java b/tools/product_config/src/com/android/build/config/DumpConfigParser.java index b954f3275..94bf20505 100644 --- a/tools/product_config/src/com/android/build/config/DumpConfigParser.java +++ b/tools/product_config/src/com/android/build/config/DumpConfigParser.java @@ -1,4 +1,3 @@ - /* * Copyright (C) 2020 The Android Open Source Project * @@ -183,7 +182,8 @@ public class DumpConfigParser { // There is already a file in progress, so add another var block to that. block = new MakeConfig.Block(MakeConfig.BlockType.INHERIT); - block.setInheritedFile(inheritedFile); + // TODO: Make dumpconfig.mk also output a Position for inherit-product + block.setInheritedFile(new Str(inheritedFile)); configFile.addBlock(block); if (DEBUG) { @@ -240,8 +240,8 @@ public class DumpConfigParser { + " Saw: " + blockType); } - // Add the value to the block in progress - block.addValue(varName, new Str(pos, varValue)); + // Add the variable to the block in progress + block.addVar(varName, new Str(pos, varValue)); } else { if (DEBUG) { System.out.print("# "); diff --git a/tools/product_config/src/com/android/build/config/Errors.java b/tools/product_config/src/com/android/build/config/Errors.java index 9290b72d7..92a4b3054 100644 --- a/tools/product_config/src/com/android/build/config/Errors.java +++ b/tools/product_config/src/com/android/build/config/Errors.java @@ -52,4 +52,11 @@ public class Errors extends ErrorReporter { public final Category ERROR_DUMPCONFIG = new Category(5, false, Level.ERROR, "Error parsing the output of kati and dumpconfig.mk."); + public final Category WARNING_VARIABLE_RECURSION = new Category(6, true, Level.WARNING, + "Possible unsupported variable recursion."); + + // This could be a warning, but it's very likely that the data is corrupted somehow + // if we're seeing this. + public final Category ERROR_IMPROPER_PRODUCT_VAR_MARKER = new Category(7, true, Level.ERROR, + "Bad input from dumpvars causing corrupted product variables."); } diff --git a/tools/product_config/src/com/android/build/config/GenericConfig.java b/tools/product_config/src/com/android/build/config/GenericConfig.java new file mode 100644 index 000000000..2ee273564 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/GenericConfig.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Language-agnostic representation of a configuration statement. + */ +public class GenericConfig extends ConfigBase { + /** + * The config files that were imported in this config pass. + */ + protected final TreeMap mConfigFiles = new TreeMap(); + + /** + * A configuration file. + */ + public static class ConfigFile { + /** + * The name of the file, relative to the tree root. + */ + private final String mFilename; + + /** + * Sections of variable definitions and import statements. Product config + * files will always have at least one block. + */ + private final ArrayList mStatements = new ArrayList(); + + public ConfigFile(String filename) { + mFilename = filename; + } + + public String getFilename() { + return mFilename; + } + + public void addStatement(Statement statement) { + mStatements.add(statement); + } + + public ArrayList getStatements() { + return mStatements; + } + } + + /** + * Base class for statements that appear in config files. + */ + public static class Statement { + } + + /** + * A variable assignment. + */ + public static class Assign extends Statement { + private final String mVarName; + private final List mValue; + + /** + * Assignment of a single value + */ + public Assign(String varName, Str value) { + mVarName = varName; + mValue = new ArrayList(); + mValue.add(value); + } + + /** + * Assignment referencing a previous value. + * VAR := $(1) $(VAR) $(2) $(VAR) $(3) + */ + public Assign(String varName, List value) { + mVarName = varName; + mValue = value; + } + + public String getName() { + return mVarName; + } + + public List getValue() { + return mValue; + } + } + + /** + * An $(inherit-product FILENAME) statement + */ + public static class Inherit extends Statement { + private final Str mFilename; + + public Inherit(Str filename) { + mFilename = filename; + } + + public Str getFilename() { + return mFilename; + } + } + + /** + * Adds the given config file. Returns any one previously added, or null. + */ + public ConfigFile addConfigFile(ConfigFile file) { + return mConfigFiles.put(file.getFilename(), file); + } + + public TreeMap getFiles() { + return mConfigFiles; + } +} diff --git a/tools/product_config/src/com/android/build/config/Main.java b/tools/product_config/src/com/android/build/config/Main.java index 81d9e7b4f..7417fc7dc 100644 --- a/tools/product_config/src/com/android/build/config/Main.java +++ b/tools/product_config/src/com/android/build/config/Main.java @@ -18,6 +18,7 @@ package com.android.build.config; import java.util.List; import java.util.Map; +import java.util.TreeSet; public class Main { private final Errors mErrors; @@ -46,6 +47,14 @@ public class Main { System.out.println("===================="); makeConfig.printToStream(System.out); + ConvertMakeToGenericConfig m2g = new ConvertMakeToGenericConfig(mErrors); + GenericConfig generic = m2g.convert(makeConfig); + + System.out.println("======================"); + System.out.println("REGENERATED MAKE FILES"); + System.out.println("======================"); + MakeWriter.write(System.out, generic, 0); + // TODO: Run kati and extract the variables and convert all that into starlark files. // TODO: Run starlark with all the generated ones and the hand written ones. diff --git a/tools/product_config/src/com/android/build/config/MakeConfig.java b/tools/product_config/src/com/android/build/config/MakeConfig.java index 300b65503..dda0db9b6 100644 --- a/tools/product_config/src/com/android/build/config/MakeConfig.java +++ b/tools/product_config/src/com/android/build/config/MakeConfig.java @@ -70,7 +70,7 @@ public class MakeConfig extends ConfigBase { public static class Block { private final BlockType mBlockType; private final TreeMap mValues = new TreeMap(); - private String mInheritedFile; + private Str mInheritedFile; public Block(BlockType blockType) { mBlockType = blockType; @@ -80,19 +80,23 @@ public class MakeConfig extends ConfigBase { return mBlockType; } - public void addValue(String varName, Str varValue) { + public void addVar(String varName, Str varValue) { mValues.put(varName, varValue); } - public TreeMap getValues() { + public Str getVar(String varName) { + return mValues.get(varName); + } + + public TreeMap getVars() { return mValues; } - public void setInheritedFile(String filename) { + public void setInheritedFile(Str filename) { mInheritedFile = filename; } - public String getInheritedFile() { + public Str getInheritedFile() { return mInheritedFile; } } @@ -148,7 +152,7 @@ public class MakeConfig extends ConfigBase { out.println(" inherited: " + block.getInheritedFile()); } out.println(" values: {"); - for (Map.Entry var: block.getValues().entrySet()) { + for (Map.Entry var: block.getVars().entrySet()) { if (!var.getKey().equals("PRODUCT_PACKAGES")) { continue; } diff --git a/tools/product_config/src/com/android/build/config/MakeWriter.java b/tools/product_config/src/com/android/build/config/MakeWriter.java new file mode 100644 index 000000000..8c79c4637 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/MakeWriter.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.io.PrintStream; +import java.util.List; + +public class MakeWriter { + public static final int FLAG_WRITE_HEADER = 1; + public static final int FLAG_WRITE_ANNOTATIONS = 1 << 1; + + private final boolean mWriteHeader; + private final boolean mWriteAnnotations; + + public static void write(PrintStream out, GenericConfig config, int flags) { + (new MakeWriter(flags)).write(out, config); + } + + private MakeWriter(int flags) { + mWriteHeader = (flags & FLAG_WRITE_HEADER) != 0; + mWriteAnnotations = (flags & FLAG_WRITE_ANNOTATIONS) != 0; + } + + private void write(PrintStream out, GenericConfig config) { + for (GenericConfig.ConfigFile file: config.getFiles().values()) { + out.println("---------------------------------------------------------"); + out.println("FILE: " + file.getFilename()); + out.println("---------------------------------------------------------"); + writeFile(out, config, file); + out.println(); + } + } + + private void writeFile(PrintStream out, GenericConfig config, GenericConfig.ConfigFile file) { + if (mWriteHeader) { + out.println("# This file is generated by the product_config tool"); + } + for (GenericConfig.Statement statement: file.getStatements()) { + if (statement instanceof GenericConfig.Assign) { + writeAssign(out, config, (GenericConfig.Assign)statement); + } else if (statement instanceof GenericConfig.Inherit) { + writeInherit(out, (GenericConfig.Inherit)statement); + } else { + throw new RuntimeException("Unexpected Statement: " + statement); + } + } + } + + private void writeAssign(PrintStream out, GenericConfig config, + GenericConfig.Assign statement) { + final List values = statement.getValue(); + final int size = values.size(); + final String varName = statement.getName(); + Position pos = null; + if (size == 0) { + return; + } else if (size == 1) { + // Plain := + final Str value = values.get(0); + out.print(varName + " := " + value); + pos = value.getPosition(); + } else if (size == 2 && values.get(0).toString().length() == 0) { + // Plain += + final Str value = values.get(1); + out.print(varName + " += " + value); + pos = value.getPosition(); + } else { + // Write it out the long way + out.print(varName + " := " + values.get(0)); + for (int i = 1; i < size; i++) { + out.print("$(" + varName + ") " + values.get(i)); + pos = values.get(i).getPosition(); + } + } + if (mWriteAnnotations) { + out.print(" # " + config.getVarType(varName) + " " + pos); + } + out.println(); + } + + private void writeInherit(PrintStream out, GenericConfig.Inherit statement) { + final Str filename = statement.getFilename(); + out.print("$(call inherit-product " + filename + ")"); + if (mWriteAnnotations) { + out.print(" # " + filename.getPosition()); + } + out.println(); + } +} diff --git a/tools/product_config/src/com/android/build/config/Str.java b/tools/product_config/src/com/android/build/config/Str.java index 7dbe2e5d6..9c345a632 100644 --- a/tools/product_config/src/com/android/build/config/Str.java +++ b/tools/product_config/src/com/android/build/config/Str.java @@ -16,6 +16,9 @@ package com.android.build.config; +import java.util.ArrayList; +import java.util.List; + /** * A String and a Position, where it came from in source code. */ @@ -64,4 +67,12 @@ public class Str { public int hashCode() { return mValue.hashCode(); } + + public static ArrayList toList(Position pos, List list) { + final ArrayList result = new ArrayList(list.size()); + for (String s: list) { + result.add(new Str(pos, s)); + } + return result; + } }