Increase parallelization

This commit is contained in:
Michael Pfaff 2022-09-08 12:22:31 -04:00
parent 6b8aad48ee
commit b418015c02
Signed by: michael
GPG Key ID: CF402C4A012AA9D4
6 changed files with 138 additions and 108 deletions

View File

@ -45,6 +45,7 @@ import javax.lang.model.util.ElementFilter;
import javax.tools.Diagnostic;
import static org.lwjgl.util.generator.Utils.getRequiredOption;
import static org.lwjgl.util.generator.Utils.getOptionalOption;
/**
* Generator tool for creating the java classes and native code from an
@ -68,8 +69,8 @@ public class GeneratorProcessor extends AbstractProcessor {
}
Map<String, String> options = processingEnv.getOptions();
String typemap_classname = getRequiredOption(options, "typemap", "the name of a TypeMap class.");
Path gen_java_path = Path.of(getRequiredOption(options, "genJavaPath", "a path."));
Path gen_native_path = Path.of(getRequiredOption(options, "genNativePath", "a path."));
Path gen_java_path = getOptionalOption(options, "genJavaPath", "a path.", Path::of);
Path gen_native_path = getOptionalOption(options, "genNativePath", "a path.", Path::of);
boolean generate_error_checks = options.containsKey("generatechecks") && !options.containsKey("nogeneratechecks");
boolean context_specific = options.containsKey("contextspecific");

View File

@ -31,6 +31,7 @@
*/
package org.lwjgl.util.generator;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -46,8 +47,11 @@ import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import javax.annotation.processing.Messager;
import javax.annotation.processing.ProcessingEnvironment;
import javax.lang.model.element.*;
@ -59,6 +63,9 @@ import javax.tools.Diagnostic.Kind;
import javax.tools.FileObject;
import javax.tools.StandardLocation;
import static org.lwjgl.util.generator.Utils.await;
import static org.lwjgl.util.generator.Utils.spawn;
/**
* Generator visitor for the generator tool
*
@ -198,7 +205,7 @@ public class GeneratorVisitor extends ElementKindVisitor6<Void, Void> {
}
}
private static void generateMethodsNativePointers(PrintWriter writer, Collection<? extends ExecutableElement> methods) {
private static void generateMethodsNativePointers(PrintWriter writer, Collection<ExecutableElement> methods) {
for ( ExecutableElement method : methods ) {
if ( method.getAnnotation(Alternate.class) == null ) {
generateMethodNativePointers(writer, method);
@ -213,7 +220,7 @@ public class GeneratorVisitor extends ElementKindVisitor6<Void, Void> {
writer.append(Utils.getTypedefName(method)).append(' ').append(method.getSimpleName()).append(";\n");
}
private void generateJavaSource(TypeElement d, PrintWriter java_writer) throws IOException {
private void generateJavaSource(TypeElement d, PrintWriter java_writer, Collection<ExecutableElement> methods, Collection<VariableElement> fields) throws IOException {
java_writer.append("/* MACHINE GENERATED FILE, DO NOT EDIT */\n\n");
java_writer.append("package ").append(env.getElementUtils().getPackageOf(d).getQualifiedName().toString()).append(";\n\n");
java_writer.append("import org.lwjgl.*;\n"
@ -243,13 +250,13 @@ public class GeneratorVisitor extends ElementKindVisitor6<Void, Void> {
java_writer.append(" extends ").append(Utils.getSimpleClassName(env.getElementUtils().getTypeElement(super_interface.toString())));
}
java_writer.append(" {\n");
FieldsGenerator.generateFields(env, java_writer, Utils.getFields(d));
FieldsGenerator.generateFields(env, java_writer, fields);
java_writer.append('\n');
if ( is_final ) {
// Write private constructor to avoid instantiation
java_writer.append("\tprivate ").append(Utils.getSimpleClassName(d)).append("() {}\n");
}
if ( Utils.getMethods(d).size() > 0 && !context_specific ) {
if ( !methods.isEmpty() && !context_specific ) {
java_writer.append('\n');
java_writer.append("\tstatic native void ").append(Utils.STUB_INITIALIZER_NAME).append("() throws LWJGLException;\n");
}
@ -257,22 +264,22 @@ public class GeneratorVisitor extends ElementKindVisitor6<Void, Void> {
java_writer.append("}");
}
private void generateNativeSource(TypeElement d, long startTime) throws IOException {
private void generateNativeSource(TypeElement d, Collection<ExecutableElement> methods, long startTime) throws IOException {
if ( d.getKind().equals(ElementKind.ANNOTATION_TYPE) ) {
return;
}
String qualified_interface_name = Utils.getQualifiedClassName(d);
StringWriter native_writer1 = new StringWriter();
ByteArrayOutputStream native_writer1 = new ByteArrayOutputStream();
PrintWriter native_writer = new PrintWriter(native_writer1);
native_writer.append("/* MACHINE GENERATED FILE, DO NOT EDIT */\n\n");
native_writer.append("#include <jni.h>\n");
type_map.printNativeIncludes(native_writer);
native_writer.append('\n');
TypedefsGenerator.generateNativeTypedefs(type_map, native_writer, Utils.getMethods(d));
TypedefsGenerator.generateNativeTypedefs(type_map, native_writer, methods);
native_writer.append('\n');
if (!context_specific) {
generateMethodsNativePointers(native_writer, Utils.getMethods(d));
generateMethodsNativePointers(native_writer, methods);
native_writer.append('\n');
}
NativeMethodStubsGenerator.generateNativeMethodStubs(env, type_map, native_writer, d, generate_error_checks, context_specific);
@ -288,52 +295,45 @@ public class GeneratorVisitor extends ElementKindVisitor6<Void, Void> {
native_writer.append("(env, clazz, num_functions, functions);\n");
native_writer.append("}");
}
saveGeneratedCSource(env.getMessager(), this.gen_native_path, d, native_writer1.toString(), startTime);
saveGeneratedCSource(env.getMessager(), this.gen_native_path, d, spawn((Callable<byte[]>) () -> native_writer1.toByteArray()), startTime);
}
private static boolean isFileExistingAndIdentical(Messager messager, Path file, String expected) throws IOException {
if (!Files.exists(file)) return false;
try (Reader existingIs = Files.newBufferedReader(file, Charset.forName("UTF-8"))) {
int i = 0;
int c;
do {
c = existingIs.read();
if (c == -1) return i == expected.length();
if (expected.length() <= i || c != expected.charAt(i++)) {
if (expected.length() > (i-1)) {
messager.printMessage(Kind.NOTE, "mismatch at " + i + ": '" + ((char) c) + '\'');
} else {
messager.printMessage(Kind.NOTE, "mismatch at " + i + ": EOF in expected");
}
return false;
}
} while (c != -1);
if (i != expected.length()) {
messager.printMessage(Kind.NOTE, "mismatch at " + i + ": EOF in existing");
return false;
}
}
return true;
}
private void doJavaGen(TypeElement e) {
private void doJavaGen(TypeElement e, Collection<ExecutableElement> methods, Collection<VariableElement> fields) {
long startTime = System.currentTimeMillis();
StringWriter java_writer = new StringWriter();
ByteArrayOutputStream java_writer = new ByteArrayOutputStream();
try {
generateJavaSource(e, new PrintWriter(java_writer));
generateJavaSource(e, new PrintWriter(java_writer), methods, fields);
} catch (IOException ex) {
throw new RuntimeException("Failed to generate the Java sources for " + e, ex);
}
saveGeneratedJavaSource(env.getMessager(), this.gen_java_path, e, java_writer.toString(), startTime);
saveGeneratedJavaSource(env.getMessager(), this.gen_java_path, e, spawn((Callable<byte[]>) () -> java_writer.toByteArray()), startTime);
}
private void doNativeGen(TypeElement e, Collection<ExecutableElement> methods) {
long startTime = System.currentTimeMillis();
boolean noNative = true;
for (ExecutableElement method : methods) {
Alternate alt_annotation = method.getAnnotation(Alternate.class);
if ((alt_annotation == null || alt_annotation.nativeAlt()) && method.getAnnotation(Reuse.class) == null) {
noNative = false;
break;
}
}
if (!noNative) {
try {
generateNativeSource(e, methods, startTime);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}
@Override
public Void visitTypeAsInterface(TypeElement e, Void p) {
long startTime = System.currentTimeMillis();
final Collection<? extends ExecutableElement> methods = Utils.getMethods(e);
if (methods.isEmpty() && Utils.getFields(e).isEmpty()) {
final Collection<ExecutableElement> methods = Utils.getMethods(e);
final Collection<VariableElement> fields = Utils.getFields(e);
if (methods.isEmpty() && fields.isEmpty()) {
return DEFAULT_VALUE;
}
@ -347,75 +347,63 @@ public class GeneratorVisitor extends ElementKindVisitor6<Void, Void> {
env.getMessager().printMessage(Diagnostic.Kind.NOTE, "Validated " + e + " in " + elapsed + " ms.");
}
if (methods.size() > 0) {
Thread javaGenThread = new Thread((Runnable) () -> {
doJavaGen(e);
});
javaGenThread.start();
if (methods.size() > 0 && this.gen_native_path != null) {
if (this.gen_java_path != null) {
Future<Void> javaGenThread = spawn(() -> {
doJavaGen(e, methods, fields);
return null;
});
long startTime1 = System.currentTimeMillis();
boolean noNative = true;
for (ExecutableElement method : methods) {
Alternate alt_annotation = method.getAnnotation(Alternate.class);
if ((alt_annotation == null || alt_annotation.nativeAlt()) && method.getAnnotation(Reuse.class) == null) {
noNative = false;
break;
}
}
if (!noNative) {
try {
generateNativeSource(e, startTime1);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
doNativeGen(e, methods);
try {
javaGenThread.join();
} catch (Throwable ex) {
throw new RuntimeException(ex);
await(javaGenThread);
} else {
doNativeGen(e, methods);
}
} else {
doJavaGen(e);
} else if (this.gen_java_path != null) {
doJavaGen(e, methods, fields);
}
return DEFAULT_VALUE;
}
public static void saveGeneratedJavaSource(Messager messager, Path gen_java_path, TypeElement e, String content, long startTime) {
public static void saveGeneratedJavaSource(Messager messager, Path gen_java_path, TypeElement e, Future<byte[]> content, long startTime) {
String qualified_interface_name = Utils.getQualifiedClassName(e);
saveGeneratedJavaSource(messager, gen_java_path, qualified_interface_name, content, startTime);
}
public static void saveGeneratedJavaSource(Messager messager, Path gen_java_path, String qualified_interface_name, String content, long startTime) {
public static void saveGeneratedJavaSource(Messager messager, Path gen_java_path, String qualified_interface_name, Future<byte[]> content, long startTime) {
final Path output = gen_java_path.resolve(qualified_interface_name.replace('.', '/') + ".java");
saveGeneratedSource(messager, "java", qualified_interface_name, output, content, startTime);
}
public static void saveGeneratedCSource(Messager messager, Path gen_native_path, TypeElement e, String content, long startTime) {
public static void saveGeneratedCSource(Messager messager, Path gen_native_path, TypeElement e, Future<byte[]> content, long startTime) {
String qualified_interface_name = Utils.getQualifiedClassName(e);
final Path output = gen_native_path.resolve(Utils.getNativeQualifiedName(qualified_interface_name) + ".c");
saveGeneratedSource(messager, "c", qualified_interface_name, output, content, startTime);
}
public static void saveGeneratedSource(Messager messager, String type, String name, Path output, String newStr, long startTime) {
public static void saveGeneratedSource(Messager messager, String type, String name, Path output, Future<byte[]> newStr, long startTime) {
try {
if (isFileExistingAndIdentical(messager, output, newStr)) {
long elapsed = System.currentTimeMillis() - startTime;
messager.printMessage(Diagnostic.Kind.NOTE, "Skipped identical '." + type + "' " + name + " at " + output + " in " + elapsed + " ms.");
return;
if (Files.exists(output)) {
Future<byte[]> actual = spawn(() -> Files.readAllBytes(output));
byte[] expected = await(newStr);
if (Arrays.equals(expected, await(actual))) {
long elapsed = System.currentTimeMillis() - startTime;
messager.printMessage(Diagnostic.Kind.NOTE, "Skipped identical '." + type + "' " + name + " at " + output + " in " + elapsed + " ms.");
return;
}
} else {
Files.createDirectories(output.getParent());
}
Files.createDirectories(output.getParent());
Files.deleteIfExists(output);
Files.createFile(output);
} catch (IOException ex) {
throw new RuntimeException("Failed to create the output file for " + name, ex);
} catch (IOException e) {
throw new RuntimeException("Failed to create the output file for " + name, e);
}
try (Writer java_file_writer = new FileWriter(output.toFile())) {
java_file_writer.write(newStr);
java_file_writer.flush();
} catch (Exception ex) {
throw new RuntimeException(ex);
try {
Files.write(output, await(newStr));
} catch (Exception e) {
throw new RuntimeException(e);
}
long elapsed = System.currentTimeMillis() - startTime;
if (elapsed > 5) {

View File

@ -49,6 +49,10 @@ import java.io.PrintWriter;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.regex.Pattern;
import javax.annotation.processing.ProcessingEnvironment;
import javax.lang.model.element.*;
@ -73,7 +77,7 @@ public class Utils {
public static final String CACHED_BUFFER_NAME = "old_buffer";
private static final String OVERLOADED_METHOD_PREFIX = "n";
public static String getRequiredOption(Map<String, String> options, String name, String description) {
public static String getRequiredOption(Map<String, String> options, String name, String description) {
String optionValue = options.get(name);
if (optionValue == null) {
throw new IllegalArgumentException("No value specified for the '" + name + "' option. Expected " + description);
@ -81,6 +85,16 @@ public class Utils {
return optionValue;
}
public static <T> T getRequiredOption(Map<String, String> options, String name, String description, Function<String, T> adapter) {
return adapter.apply(getRequiredOption(options, name, description));
}
public static <T> T getOptionalOption(Map<String, String> options, String name, String description, Function<String, T> adapter) {
String optionValue = options.get(name);
if (optionValue == null) return null;
return adapter.apply(optionValue);
}
public static String getTypedefName(ExecutableElement method) {
Alternate alt_annotation = method.getAnnotation(Alternate.class);
return (alt_annotation == null ? method.getSimpleName() : alt_annotation.value()) + TYPEDEF_POSTFIX;
@ -518,4 +532,20 @@ public class Utils {
return ElementFilter.methodsIn(d.getEnclosedElements());
}
/**
* Awaits the future and throws unchecked exceptions.
*/
public static <T> T await(Future<T> future) {
try {
return future.get();
} catch (java.util.concurrent.ExecutionException e) {
throw new RuntimeException(e.getCause());
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
public static <T> Future<T> spawn(Callable<T> f) {
return ForkJoinPool.commonPool().submit(f);
}
}

View File

@ -35,6 +35,7 @@ import org.lwjgl.PointerWrapper;
import org.lwjgl.opencl.CLDevice;
import org.lwjgl.opencl.CLPlatform;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
@ -42,6 +43,7 @@ import java.nio.file.Path;
import java.lang.annotation.Annotation;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import javax.annotation.processing.*;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.TypeElement;
@ -49,6 +51,8 @@ import javax.lang.model.util.ElementFilter;
import static org.lwjgl.util.generator.GeneratorVisitor.saveGeneratedJavaSource;
import static org.lwjgl.util.generator.Utils.getRequiredOption;
import static org.lwjgl.util.generator.Utils.await;
import static org.lwjgl.util.generator.Utils.spawn;
/**
* Generator tool for creating the OpenCL capabilities classes
@ -112,7 +116,7 @@ public class CLGeneratorProcessor extends AbstractProcessor {
private void generateCLCapabilitiesSource(Path genJavaPath, Set<TypeElement> templates) throws IOException {
long startTime = System.currentTimeMillis();
StringWriter writer1 = new StringWriter();
ByteArrayOutputStream writer1 = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(writer1);
printHeader(writer);
@ -134,12 +138,12 @@ public class CLGeneratorProcessor extends AbstractProcessor {
}
writer.println("}");
saveGeneratedJavaSource(processingEnv.getMessager(), genJavaPath, "org.lwjgl.opencl." + CLCAPS_CLASS_NAME, writer1.toString(), startTime);
saveGeneratedJavaSource(processingEnv.getMessager(), genJavaPath, "org.lwjgl.opencl." + CLCAPS_CLASS_NAME, spawn((Callable<byte[]>) () -> writer1.toByteArray()), startTime);
}
private void generateCLPDCapabilitiesSource(Path genJavaPath, Set<TypeElement> templates, final Class<? extends Annotation> capsType, final String capsName, final Class<? extends PointerWrapper> objectType, final String objectName) throws IOException {
long startTime = System.currentTimeMillis();
StringWriter writer1 = new StringWriter();
ByteArrayOutputStream writer1 = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(writer1);
printHeader(writer);
writer.println("import java.util.*;");
@ -161,6 +165,6 @@ public class CLGeneratorProcessor extends AbstractProcessor {
CLPDCapabilitiesGenerator.generateToString(writer, templates, capsType);
writer.println("}");
saveGeneratedJavaSource(processingEnv.getMessager(), genJavaPath, "org.lwjgl.opencl." + capsName, writer1.toString(), startTime);
saveGeneratedJavaSource(processingEnv.getMessager(), genJavaPath, "org.lwjgl.opencl." + capsName, spawn((Callable<byte[]>) () -> writer1.toByteArray()), startTime);
}
}

View File

@ -33,12 +33,13 @@ package org.lwjgl.util.generator.opengl;
import org.lwjgl.util.generator.Utils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Path;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import javax.annotation.processing.*;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.TypeElement;
@ -46,6 +47,8 @@ import javax.lang.model.util.ElementFilter;
import static org.lwjgl.util.generator.GeneratorVisitor.saveGeneratedJavaSource;
import static org.lwjgl.util.generator.Utils.getRequiredOption;
import static org.lwjgl.util.generator.Utils.await;
import static org.lwjgl.util.generator.Utils.spawn;
/**
* Generator tool for creating the ContextCapabilities class
@ -86,7 +89,7 @@ public class GLGeneratorProcessor extends AbstractProcessor {
long startTime = System.currentTimeMillis();
ProcessingEnvironment env = this.processingEnv;
StringWriter writer1 = new StringWriter();
ByteArrayOutputStream writer1 = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(writer1);
writer.println("/* MACHINE GENERATED FILE, DO NOT EDIT */");
@ -169,6 +172,6 @@ public class GLGeneratorProcessor extends AbstractProcessor {
writer.println("\t}");
writer.println("}");
saveGeneratedJavaSource(env.getMessager(), genJavaPath, "org.lwjgl.opengl." + Utils.CONTEXT_CAPS_CLASS_NAME, writer1.toString(), startTime);
saveGeneratedJavaSource(env.getMessager(), genJavaPath, "org.lwjgl.opengl." + Utils.CONTEXT_CAPS_CLASS_NAME, spawn((Callable<byte[]>) () -> writer1.toByteArray()), startTime);
}
}

View File

@ -35,12 +35,14 @@ import org.lwjgl.util.generator.Alternate;
import org.lwjgl.util.generator.CachedReference;
import org.lwjgl.util.generator.Utils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Path;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import javax.annotation.processing.*;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.ExecutableElement;
@ -50,6 +52,8 @@ import javax.lang.model.util.ElementFilter;
import static org.lwjgl.util.generator.GeneratorVisitor.saveGeneratedJavaSource;
import static org.lwjgl.util.generator.Utils.getRequiredOption;
import static org.lwjgl.util.generator.Utils.await;
import static org.lwjgl.util.generator.Utils.spawn;
/**
* Generator tool for creating the References class
@ -157,7 +161,7 @@ public class GLReferencesGeneratorProcessor extends AbstractProcessor {
private void generateReferencesSource(ProcessingEnvironment env, Path genJavaPath, Set<TypeElement> templates) throws IOException {
long startTime = System.currentTimeMillis();
StringWriter writer1 = new StringWriter();
ByteArrayOutputStream writer1 = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(writer1);
writer.println("/* MACHINE GENERATED FILE, DO NOT EDIT */");
writer.println();
@ -168,35 +172,35 @@ public class GLReferencesGeneratorProcessor extends AbstractProcessor {
writer.println("\t\tsuper(caps);");
writer.println("\t}");
templates.stream().parallel().filter(tmpl -> tmpl.getKind().isInterface()).map(interface_decl -> {
StringWriter writer2 = new StringWriter();
ByteArrayOutputStream writer2 = new ByteArrayOutputStream();
PrintWriter writer3 = new PrintWriter(writer2);
generateReferencesFromMethods(env, writer3, interface_decl);
return writer2.toString();
}).sequential().forEach(writer::append);
return writer2.toByteArray();
}).sequential().forEach(writer1::writeBytes);
writer.println();
writer.println("\tvoid copy(" + REFERENCES_CLASS_NAME + " " + REFERENCES_PARAMETER_NAME + ", int mask) {");
writer.println("\t\tsuper.copy(" + REFERENCES_PARAMETER_NAME + ", mask);");
writer.println("\t\tif ( (mask & GL11.GL_CLIENT_VERTEX_ARRAY_BIT) != 0 ) {");
templates.stream().parallel().filter(tmpl -> tmpl.getKind().isInterface()).map(interface_decl -> {
StringWriter writer2 = new StringWriter();
ByteArrayOutputStream writer2 = new ByteArrayOutputStream();
PrintWriter writer3 = new PrintWriter(writer2);
generateCopiesFromMethods(processingEnv, writer, interface_decl);
return writer2.toString();
}).sequential().forEach(writer::append);
return writer2.toByteArray();
}).sequential().forEach(writer1::writeBytes);
writer.println("\t\t}");
writer.println("\t}");
writer.println("\tvoid clear() {");
writer.println("\t\tsuper.clear();");
templates.stream().parallel().filter(tmpl -> tmpl.getKind().isInterface()).map(interface_decl -> {
StringWriter writer2 = new StringWriter();
ByteArrayOutputStream writer2 = new ByteArrayOutputStream();
PrintWriter writer3 = new PrintWriter(writer2);
generateClearsFromMethods(processingEnv, writer, interface_decl);
return writer2.toString();
}).sequential().forEach(writer::append);
return writer2.toByteArray();
}).sequential().forEach(writer1::writeBytes);
writer.println("\t}");
writer.println("}");
saveGeneratedJavaSource(env.getMessager(), genJavaPath, "org.lwjgl.opengl." + REFERENCES_CLASS_NAME, writer1.toString(), startTime);
saveGeneratedJavaSource(env.getMessager(), genJavaPath, "org.lwjgl.opengl." + REFERENCES_CLASS_NAME, spawn((Callable<byte[]>) () -> writer1.toByteArray()), startTime);
}
}