diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java index 39eecdb66b..5027651463 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java @@ -42,17 +42,19 @@ import java.io.*; import java.nio.file.Files; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; import java.util.zip.CRC32; import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; public class ApkBuilder { private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName()); + private final AtomicReference mBuildError = new AtomicReference<>(null); private final Config mConfig; private final ExtFile mApkDir; + private BackgroundWorker mWorker; private ApkInfo mApkInfo; private int mMinSdkVersion = 0; @@ -78,51 +80,55 @@ public ApkBuilder(Config config, ExtFile apkDir) { public void build(File outFile) throws BrutException { LOGGER.info("Using Apktool " + ApktoolProperties.getVersion()); + try { + mWorker = new BackgroundWorker(); + mApkInfo = ApkInfo.load(mApkDir); - mApkInfo = ApkInfo.load(mApkDir); + if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) { + String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion"); + mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion); + } - if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) { - String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion"); - mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion); - } + if (outFile == null) { + String outFileName = mApkInfo.apkFileName; + outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName)); + } - if (outFile == null) { - String outFileName = mApkInfo.apkFileName; - outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName)); - } + //noinspection ResultOfMethodCallIgnored + new File(mApkDir, APK_DIRNAME).mkdirs(); + File manifest = new File(mApkDir, "AndroidManifest.xml"); + File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig"); + + scheduleBuildDexFiles(); + backupManifestFile(manifest, manifestOriginal); + buildResources(); + copyLibs(); + copyOriginalFilesIfEnabled(); + mWorker.waitForFinish(); + if (mBuildError.get() != null) { + throw mBuildError.get(); + } - //noinspection ResultOfMethodCallIgnored - new File(mApkDir, APK_DIRNAME).mkdirs(); - File manifest = new File(mApkDir, "AndroidManifest.xml"); - File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig"); - - buildSources(); - buildNonDefaultSources(); - buildManifestFile(manifest, manifestOriginal); - buildResources(); - buildLibs(); - buildCopyOriginalFiles(); - buildApk(outFile); - - // we must go after the Apk is built, and copy the files in via Zip - // this is because Aapt won't add files it doesn't know (ex unknown files) - buildUnknownFiles(outFile); - - // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it - // lets restore the unedited one, to not change the original - if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) { - try { - if (new File(mApkDir, "AndroidManifest.xml").delete()) { - FileUtils.moveFile(manifestOriginal, manifest); + buildApk(outFile); + + // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it + // lets restore the unedited one, to not change the original + if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) { + try { + if (new File(mApkDir, "AndroidManifest.xml").delete()) { + FileUtils.moveFile(manifestOriginal, manifest); + } + } catch (IOException ex) { + throw new AndrolibException(ex.getMessage()); } - } catch (IOException ex) { - throw new AndrolibException(ex.getMessage()); } + LOGGER.info("Built apk into: " + outFile.getPath()); + } finally { + mWorker.shutdownNow(); } - LOGGER.info("Built apk into: " + outFile.getPath()); } - private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException { + private void backupManifestFile(File manifest, File manifestOriginal) throws AndrolibException { // If we decoded in "raw", we cannot patch AndroidManifest if (new File(mApkDir, "resources.arsc").exists()) { return; @@ -141,24 +147,17 @@ private void buildManifestFile(File manifest, File manifestOriginal) throws Andr } } - private void buildSources() throws AndrolibException { - if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) { - LOGGER.warning("Could not find sources"); - } - } - - private void buildNonDefaultSources() throws AndrolibException { + private void scheduleBuildDexFiles() throws AndrolibException { try { + mWorker.submit(() -> scheduleDexBuild("classes.dex", "smali")); + // loop through any smali_ directories for multi-dex apks Map dirs = mApkDir.getDirectory().getDirs(); for (Map.Entry directory : dirs.entrySet()) { String name = directory.getKey(); if (name.startsWith("smali_")) { String filename = name.substring(name.indexOf("_") + 1) + ".dex"; - - if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) { - LOGGER.warning("Could not find sources"); - } + mWorker.submit(() -> scheduleDexBuild(filename, name)); } } @@ -177,6 +176,19 @@ private void buildNonDefaultSources() throws AndrolibException { } } + private void scheduleDexBuild(String filename, String smali) { + try { + if (mBuildError.get() != null) { + return; + } + if (!buildSourcesRaw(filename) && !buildSourcesSmali(smali, filename)) { + LOGGER.warning("Could not find sources"); + } + } catch (AndrolibException e) { + mBuildError.compareAndSet(null, e); + } + } + private boolean buildSourcesRaw(String filename) throws AndrolibException { File working = new File(mApkDir, filename); if (!working.exists()) { @@ -214,6 +226,7 @@ private boolean buildSourcesSmali(String folder, String filename) throws Androli } private void buildResources() throws BrutException { + // create res folder, manifest file and resources.arsc if (!buildResourcesRaw() && !buildResourcesFull() && !buildManifest()) { LOGGER.warning("Could not find resources"); } @@ -375,7 +388,7 @@ private boolean buildManifest() throws BrutException { } } - private void buildLibs() throws AndrolibException { + private void copyLibs() throws AndrolibException { buildLibrary("lib"); buildLibrary("libs"); buildLibrary("kotlin"); @@ -401,7 +414,7 @@ private void buildLibrary(String folder) throws AndrolibException { } } - private void buildCopyOriginalFiles() throws AndrolibException { + private void copyOriginalFilesIfEnabled() throws AndrolibException { if (mConfig.copyOriginalFiles) { File originalDir = new File(mApkDir, "original"); if (originalDir.exists()) { @@ -427,49 +440,34 @@ private void buildCopyOriginalFiles() throws AndrolibException { } } - private void buildUnknownFiles(File outFile) throws AndrolibException { - if (mApkInfo.unknownFiles != null) { - LOGGER.info("Copying unknown files/dir..."); - - Map files = mApkInfo.unknownFiles; - File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); - boolean renamed = outFile.renameTo(tempFile); - if (!renamed) { - throw new AndrolibException("Unable to rename temporary file"); - } - - try ( - ZipFile inputFile = new ZipFile(tempFile); - ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath())) - ) { - copyExistingFiles(inputFile, actualOutput); - copyUnknownFiles(actualOutput, files); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } - - // Remove our temporary file. + private void buildApk(File outApk) throws AndrolibException { + LOGGER.info("Building apk file..."); + if (outApk.exists()) { //noinspection ResultOfMethodCallIgnored - tempFile.delete(); + outApk.delete(); + } else { + File outDir = outApk.getParentFile(); + if (outDir != null && !outDir.exists()) { + //noinspection ResultOfMethodCallIgnored + outDir.mkdirs(); + } } - } - - private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException { - // First, copy the contents from the existing outFile: - Enumeration entries = inputFile.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = new ZipEntry(entries.nextElement()); - - // We can't reuse the compressed size because it depends on compression sizes. - entry.setCompressedSize(-1); - outputFile.putNextEntry(entry); + File assetDir = new File(mApkDir, "assets"); + if (!assetDir.exists()) { + assetDir = null; + } + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) { + // zip all AAPT-generated files + ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress); - // No need to create directory entries in the final apk - if (!entry.isDirectory()) { - BrutIO.copy(inputFile, outputFile, entry); + // we must copy some files manually + // this is because Aapt won't add files it doesn't know (ex unknown files) + if (mApkInfo.unknownFiles != null) { + LOGGER.info("Copying unknown files/dir..."); + copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles); } - - outputFile.closeEntry(); + } catch (IOException | BrutException e) { + throw new AndrolibException(e); } } @@ -513,33 +511,6 @@ private void copyUnknownFiles(ZipOutputStream outputFile, Map fi } } - private void buildApk(File outApk) throws AndrolibException { - LOGGER.info("Building apk file..."); - if (outApk.exists()) { - //noinspection ResultOfMethodCallIgnored - outApk.delete(); - } else { - File outDir = outApk.getParentFile(); - if (outDir != null && !outDir.exists()) { - //noinspection ResultOfMethodCallIgnored - outDir.mkdirs(); - } - } - File assetDir = new File(mApkDir, "assets"); - if (!assetDir.exists()) { - assetDir = null; - } - zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir); - } - - private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException { - try { - ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } - } - private File[] getIncludeFiles() throws AndrolibException { UsesFramework usesFramework = mApkInfo.usesFramework; if (usesFramework == null) { diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java index 1d04985b28..ec71135419 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java @@ -32,15 +32,18 @@ import java.io.*; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; import java.util.regex.Pattern; public class ApkDecoder { private final static Logger LOGGER = Logger.getLogger(ApkDecoder.class.getName()); + private final AtomicReference mBuildError = new AtomicReference<>(null); private final Config mConfig; private final ApkInfo mApkInfo; - private int mMinSdkVersion = 0; + private volatile int mMinSdkVersion = 0; + private BackgroundWorker mWorker; private final static String SMALI_DIRNAME = "smali"; private final static String UNK_DIRNAME = "unknown"; @@ -75,6 +78,7 @@ public ApkDecoder(Config config, ExtFile apkFile) { public ApkInfo decode(File outDir) throws AndrolibException, IOException, DirectoryException { ExtFile apkFile = mApkInfo.getApkFile(); try { + mWorker = new BackgroundWorker(); if (!mConfig.forceDelete && outDir.exists()) { throw new OutDirExistsException(); } @@ -93,30 +97,6 @@ public ApkInfo decode(File outDir) throws AndrolibException, IOException, Direct LOGGER.info("Using Apktool " + ApktoolProperties.getVersion() + " on " + mApkInfo.apkFileName); - ResourcesDecoder resourcesDecoder = new ResourcesDecoder(mConfig, mApkInfo); - - if (mApkInfo.hasResources()) { - switch (mConfig.decodeResources) { - case Config.DECODE_RESOURCES_NONE: - copyResourcesRaw(outDir); - break; - case Config.DECODE_RESOURCES_FULL: - resourcesDecoder.decodeResources(outDir); - break; - } - } - - if (mApkInfo.hasManifest()) { - if (mConfig.decodeResources == Config.DECODE_RESOURCES_FULL || - mConfig.forceDecodeManifest == Config.FORCE_DECODE_MANIFEST_FULL) { - resourcesDecoder.decodeManifest(outDir); - } - else { - copyManifestRaw(outDir); - } - } - resourcesDecoder.updateApkInfo(outDir); - if (mApkInfo.hasSources()) { switch (mConfig.decodeSources) { case Config.DECODE_SOURCES_NONE: @@ -124,7 +104,7 @@ public ApkInfo decode(File outDir) throws AndrolibException, IOException, Direct break; case Config.DECODE_SOURCES_SMALI: case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES: - decodeSourcesSmali(outDir, "classes.dex"); + scheduleDecodeSourcesSmali(outDir, "classes.dex"); break; } } @@ -140,11 +120,11 @@ public ApkInfo decode(File outDir) throws AndrolibException, IOException, Direct copySourcesRaw(outDir, file); break; case Config.DECODE_SOURCES_SMALI: - decodeSourcesSmali(outDir, file); + scheduleDecodeSourcesSmali(outDir, file); break; case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES: if (file.startsWith("classes") && file.endsWith(".dex")) { - decodeSourcesSmali(outDir, file); + scheduleDecodeSourcesSmali(outDir, file); } else { copySourcesRaw(outDir, file); } @@ -155,19 +135,49 @@ public ApkInfo decode(File outDir) throws AndrolibException, IOException, Direct } } - // In case we have no resources. We should store the minSdk we pulled from the source opcode api level - if (!mApkInfo.hasResources() && mMinSdkVersion > 0) { - mApkInfo.setSdkInfoField("minSdkVersion", Integer.toString(mMinSdkVersion)); + ResourcesDecoder resourcesDecoder = new ResourcesDecoder(mConfig, mApkInfo); + + if (mApkInfo.hasResources()) { + switch (mConfig.decodeResources) { + case Config.DECODE_RESOURCES_NONE: + copyResourcesRaw(outDir); + break; + case Config.DECODE_RESOURCES_FULL: + resourcesDecoder.decodeResources(outDir); + break; + } + } + + if (mApkInfo.hasManifest()) { + if (mConfig.decodeResources == Config.DECODE_RESOURCES_FULL || + mConfig.forceDecodeManifest == Config.FORCE_DECODE_MANIFEST_FULL) { + resourcesDecoder.decodeManifest(outDir); + } + else { + copyManifestRaw(outDir); + } } + resourcesDecoder.updateApkInfo(outDir); copyRawFiles(outDir); copyUnknownFiles(outDir); recordUncompressedFiles(resourcesDecoder.getResFileMapping()); copyOriginalFiles(outDir); + mWorker.waitForFinish(); + if (mBuildError.get() != null) { + throw mBuildError.get(); + } + + // In case we have no resources. We should store the minSdk we pulled from the source opcode api level + if (!mApkInfo.hasResources() && mMinSdkVersion > 0) { + mApkInfo.setSdkInfoField("minSdkVersion", Integer.toString(mMinSdkVersion)); + } + writeApkInfo(outDir); return mApkInfo; } finally { + mWorker.shutdownNow(); try { apkFile.close(); } catch (IOException ignored) {} @@ -205,6 +215,17 @@ private void copySourcesRaw(File outDir, String filename) throws AndrolibExcepti } } + private void scheduleDecodeSourcesSmali(File outDir, String filename) { + Runnable r = () -> { + try { + decodeSourcesSmali(outDir, filename); + } catch (AndrolibException e) { + mBuildError.compareAndSet(null, new RuntimeException(e)); + } + }; + mWorker.submit(r); + } + private void decodeSourcesSmali(File outDir, String filename) throws AndrolibException { try { File smaliDir; diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java new file mode 100644 index 0000000000..5ba74fc96f --- /dev/null +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java @@ -0,0 +1,65 @@ +package brut.androlib; + +import java.util.ArrayList; +import java.util.concurrent.*; + +public class BackgroundWorker { + + private static final int THREADS_COUNT = Runtime.getRuntime().availableProcessors(); + private final ArrayList> mWorkerFutures = new ArrayList<>(); + private final ExecutorService mExecutor; + private volatile boolean mSubmitAllowed = true; + + public BackgroundWorker() { + this(THREADS_COUNT); + } + + public BackgroundWorker(int threads) { + mExecutor = Executors.newFixedThreadPool(threads); + } + + public void waitForFinish() { + checkState(); + mSubmitAllowed = false; + for (Future future : mWorkerFutures) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + mWorkerFutures.clear(); + mSubmitAllowed = true; + } + + public void clearFutures() { + mWorkerFutures.clear(); + } + + private void checkState() { + if (!mSubmitAllowed) { + throw new IllegalStateException("BackgroundWorker is not ready"); + } + } + + public void shutdownNow() { + mSubmitAllowed = false; + mExecutor.shutdownNow(); + } + + public ExecutorService getExecutor() { + return mExecutor; + } + + public void submit(Runnable task) { + checkState(); + mWorkerFutures.add(mExecutor.submit(task)); + } + + public Future submit(Callable task) { + checkState(); + Future future = mExecutor.submit(task); + mWorkerFutures.add(future); + return future; + } +} diff --git a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java index 3e71990016..361b9fb15a 100644 --- a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java +++ b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java @@ -39,15 +39,21 @@ private ZipUtils() { public static void zipFolders(final File folder, final File zip, final File assets, final Collection doNotCompress) throws BrutException, IOException { - mDoNotCompress = doNotCompress; ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(zip.toPath())); + zipFoldersPreserveStream(folder, zipOutputStream, assets, doNotCompress); + zipOutputStream.close(); + } + + public static void zipFoldersPreserveStream(final File folder, final ZipOutputStream zipOutputStream, final File assets, final Collection doNotCompress) + throws BrutException, IOException { + + mDoNotCompress = doNotCompress; zipFolders(folder, zipOutputStream); // We manually set the assets because we need to retain the folder structure if (assets != null) { processFolder(assets, zipOutputStream, assets.getPath().length() - 6); } - zipOutputStream.close(); } private static void zipFolders(final File folder, final ZipOutputStream outputStream)