diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 000000000..bd6a105f8 --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,130 @@ +{ + "creators": [ + { + "affiliation": "Leiden University", + "name": "Portegies Zwart, Simon", + "orcid": "0000-0001-5839-0302" + }, + { + "affiliation": "Leiden University", + "name": "van Elteren, Arjen" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Pelupessy, Inti" + }, + { + "affiliation": "Drexel University", + "name": "McMillan, Steve" + }, + { + "affiliation": "University of Exeter", + "name": "Rieder, Steven", + "orcid": "0000-0003-3688-5798" + }, + { + "affiliation": "Leiden University", + "name": "de Vries, Nathan" + }, + { + "affiliation": "Leiden University", + "name": "Marosvolgyi, Marcell" + }, + { + "affiliation": "Drexel University", + "name": "Whitehead, Alfred" + }, + { + "affiliation": "Drexel University", + "name": "Wall, Joshua" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Drost, Niels", + "orcid": "0000-0001-9795-7981" + }, + { + "affiliation": "Leiden University", + "name": "Jilkova, Lucie" + }, + { + "affiliation": "Leiden University", + "name": "Martinez Barbosa, Carmen" + }, + { + "affiliation": "Leiden University", + "name": "van der Helm, Edwin" + }, + { + "affiliation": "Leiden University", + "name": "Beedorf, Jeroen" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Bos, Patrick" + }, + { + "affiliation": "Leiden University", + "name": "Boekholt, Tjarda" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "van Werkhoven, Ben" + }, + { + "affiliation": "Leiden University", + "name": "Wijnen, Thomas" + }, + { + "affiliation": "Institute for Advanced Study", + "name": "Hamers, Adrian" + }, + { + "affiliation": "Booking.com", + "name": "Caputo, Daniel" + }, + { + "affiliation": "Leiden University", + "name": "Ferrari, Guilherme" + }, + { + "affiliation": "University of Amsterdam", + "name": "Toonen, Silvia" + }, + { + "affiliation": "Nvidia", + "name": "Gaburov, Evghenii" + }, + { + "affiliation": "TNO", + "name": "Paardekooper, Jan-Pieter" + }, + { + "affiliation": "University of Cambridge", + "name": "Janes, Jurgen" + }, + { + "affiliation": "University of Groningen", + "name": "Punzo, Davide" + }, + { + "affiliation": "Quintel Intellegence", + "name": "Kruip, Chael" + }, + { + "affiliation": "Georgia Institute of Technology", + "name": "Altay, Gabriel" + } + ], + "description": "A Python framework to combine existing astrophysical simulation codes in numerical experiments. With AMUSE you can simulate objects such as star clusters, proto-planetary disks and galaxies.", + "keywords": [ + "astronomy", + "multi-model simulation", + "model coupling", + "physics" + ], + "license": { + "id": "Apache-2.0" + }, + "title": "AMUSE: the Astrophysical Multipurpose Software Environment" +} diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..f6ad6824d --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,137 @@ +# YAML 1.2 +# to generate .zenodo json: cffconvert -ig -f zenodo -of .zenodo.json +# note we have omitted date released & version +--- +abstract: "A Python framework to combine existing astrophysical simulation codes in numerical experiments. With AMUSE you can simulate objects such as star clusters, proto-planetary disks and galaxies." +authors: + - + affiliation: "Leiden University" + family-names: "Portegies Zwart" + given-names: Simon + orcid: "https://orcid.org/0000-0001-5839-0302" + - + affiliation: "Leiden University" + family-names: Elteren + given-names: Arjen + name-particle: van + - + affiliation: "Netherlands eScience Center" + family-names: Pelupessy + given-names: Inti + - + affiliation: "Drexel University" + family-names: McMillan + given-names: Steve + - + affiliation: "University of Exeter" + family-names: Rieder + given-names: Steven + orcid: "https://orcid.org/0000-0003-3688-5798" + - + affiliation: "Leiden University" + family-names: de Vries + given-names: Nathan + - + affiliation: "Leiden University" + family-names: Marosvolgyi + given-names: Marcell + - + affiliation: "Drexel University" + family-names: Whitehead + given-names: Alfred + - + affiliation: "Drexel University" + family-names: Wall + given-names: Joshua + - + affiliation: "Netherlands eScience Center" + family-names: Drost + given-names: Niels + orcid: "https://orcid.org/0000-0001-9795-7981" + - + affiliation: "Leiden University" + family-names: Jilkova + given-names: Lucie + - + affiliation: "Leiden University" + family-names: Martinez Barbosa + given-names: Carmen + - + affiliation: "Leiden University" + family-names: van der Helm + given-names: Edwin + - + affiliation: "Leiden University" + family-names: Beedorf + given-names: Jeroen + - + affiliation: "Netherlands eScience Center" + family-names: Bos + given-names: Patrick + - + affiliation: "Leiden University" + family-names: Boekholt + given-names: Tjarda + - + affiliation: "Netherlands eScience Center" + family-names: Werkhoven + given-names: Ben + name-particle: van + - + affiliation: "Leiden University" + family-names: Wijnen + given-names: Thomas + - + affiliation: "Institute for Advanced Study" + family-names: Hamers + given-names: Adrian + - + affiliation: "Booking.com" + family-names: Caputo + given-names: Daniel + - + affiliation: "Leiden University" + family-names: Ferrari + given-names: Guilherme + - + affiliation: "University of Amsterdam" + family-names: Toonen + given-names: Silvia + - + affiliation: "Nvidia" + family-names: Gaburov + given-names: Evghenii + - + affiliation: "TNO" + family-names: Paardekooper + given-names: Jan-Pieter + - + affiliation: "University of Cambridge" + family-names: Janes + given-names: Jurgen + - + affiliation: "University of Groningen" + family-names: Punzo + given-names: Davide + - + affiliation: "Quintel Intellegence" + family-names: Kruip + given-names: Chael + - + affiliation: "Georgia Institute of Technology" + family-names: Altay + given-names: Gabriel + + +cff-version: "1.0.3" +doi: "10.5281/zenodo.1435860" +keywords: + - astronomy + - "multi-model simulation" + - "model coupling" + - physics +license: "Apache-2.0" +message: "This is the software citation for the framework code. If you publish work with AMUSE please cite also the code papers of the components used." +repository-code: "https://github.com/amusecode/amuse" +title: "AMUSE: the Astrophysical Multipurpose Software Environment" +... diff --git a/examples/textbook/gravity_drifter.py b/examples/textbook/gravity_drifter.py index 751439c3c..99190d69d 100644 --- a/examples/textbook/gravity_drifter.py +++ b/examples/textbook/gravity_drifter.py @@ -1,7 +1,7 @@ -from amuse.lab import * import numpy from amuse.lab import * from amuse.units import units +from amuse.units import quantities from amuse.units import constants from amuse.units import nbody_system from amuse.ext.bridge import bridge @@ -146,7 +146,7 @@ def evolve_cluster_in_galaxy(N, W0, Rinit, tend, timestep, M, R): system = bridge(verbose=False) system.add_system(cluster_code, (galaxy_code,)) - times = numpy.arange(0|units.Myr, tend, 100*timestep) + times = quantities.arange(0|units.Myr, tend, 100*timestep) for i,t in enumerate(times): print "Time=", t.in_(units.Myr) system.evolve_model(t, timestep=timestep) diff --git a/examples/textbook/gravity_gravity.py b/examples/textbook/gravity_gravity.py index 5d08e01c6..ba8050bb4 100644 --- a/examples/textbook/gravity_gravity.py +++ b/examples/textbook/gravity_gravity.py @@ -1,6 +1,7 @@ #from __future__ import print_function import numpy from amuse.units import units +from amuse.units import quantities from amuse.units import constants from amuse.units import nbody_system from amuse.ext.bridge import bridge @@ -97,7 +98,7 @@ def evolve_cluster_in_galaxy(N, W0, Rinit, tend, timestep, M, R): system.add_system(galaxy_code, (cluster_code,)) system.timestep = 0.1*timestep - times=numpy.arange(0|units.Myr, tend, timestep) + times = quantities.arange(0|units.Myr, tend, timestep) for i,t in enumerate(times): print "Time=", t.in_(units.Myr) channe_to_galaxy.copy() diff --git a/examples/textbook/gravity_potential.py b/examples/textbook/gravity_potential.py index 473adce02..b780b60f3 100644 --- a/examples/textbook/gravity_potential.py +++ b/examples/textbook/gravity_potential.py @@ -1,6 +1,7 @@ #from __future__ import print_function import numpy from amuse.units import units +from amuse.units import quantities from amuse.units import constants from amuse.units import nbody_system from amuse.ext.bridge import bridge @@ -97,7 +98,7 @@ def evolve_cluster_in_galaxy(N, W0, Rinit, tend, timestep, M, R): system = bridge(verbose=False) system.add_system(cluster_code, (galaxy_code,)) - times = numpy.arange(0|units.Myr, tend, timestep) + times = quantities.arange(0|units.Myr, tend, timestep) for i,t in enumerate(times): system.evolve_model(t,timestep=timestep) diff --git a/examples/textbook/two_body_bridge.py b/examples/textbook/two_body_bridge.py index aff3c8393..03f203e5e 100644 --- a/examples/textbook/two_body_bridge.py +++ b/examples/textbook/two_body_bridge.py @@ -24,7 +24,7 @@ def main(): gravity.add_system(planet_gravity, (star_gravity,) ) ###BOOKLISTSTOP### - write_set_to_file(ss, filename, 'hdf5', append_to_file=Fale) + write_set_to_file(ss, filename, 'hdf5', append_to_file=False) Etot_init = gravity.kinetic_energy + gravity.potential_energy Etot_prev = Etot_init diff --git a/lib/forsockets/forsockets.c b/lib/forsockets/forsockets.c index 3b08831fb..6ac6c5ec9 100644 --- a/lib/forsockets/forsockets.c +++ b/lib/forsockets/forsockets.c @@ -11,6 +11,7 @@ #include #include #include + #include #endif int32_t socketfd; diff --git a/src/amuse/community/distributed/src/src/nl/esciencecenter/amuse/distributed/AmuseMessage.java b/src/amuse/community/distributed/src/src/nl/esciencecenter/amuse/distributed/AmuseMessage.java index df6a3444c..ef744f497 100644 --- a/src/amuse/community/distributed/src/src/nl/esciencecenter/amuse/distributed/AmuseMessage.java +++ b/src/amuse/community/distributed/src/src/nl/esciencecenter/amuse/distributed/AmuseMessage.java @@ -133,8 +133,8 @@ public AmuseMessage() { stringHeaderBytes = ByteBuffer.allocateDirect(0); stringBytes = new ByteBuffer[0]; - allButStringByteBuffers = new ByteBuffer[] { headerBytes, intBytes, longBytes, floatBytes, doubleBytes, booleanBytes, - stringHeaderBytes }; + allButStringByteBuffers = new ByteBuffer[] { headerBytes, intBytes, longBytes, floatBytes, doubleBytes, + booleanBytes, stringHeaderBytes }; // no string buffers yet byteBuffers = allButStringByteBuffers; @@ -189,7 +189,7 @@ public long getDataSize() { result += getBooleanCount() * SIZEOF_BOOLEAN; for (int i = 0; i < getStringCount(); i++) { - result += stringHeader.get(i); + result += stringHeader.get(i)+1 ; // account for zero } return result; @@ -302,10 +302,11 @@ public void setError(String error) { stringHeader.put(0, bytes.length); - ensureStringsCapacity(); + ensureStringsCapacity(0); stringBytes[0].clear(); stringBytes[0].put(bytes); + stringBytes[0].put( (byte) 0); // add extra zero } catch (UnsupportedEncodingException e) { System.err.println("could not set error: " + e); stringHeader.put(0, 0); @@ -335,17 +336,22 @@ public void addString(String value) { byte[] bytes; try { - bytes = value.getBytes("UTF-8"); + if (value == null) { + //set null values to an empty string + bytes = new String().getBytes("UTF-8"); + } else { + bytes = value.getBytes("UTF-8"); + } // set length of string in header stringHeader.put(position, bytes.length); // make sure there is space for the string - ensureStringsCapacity(); + ensureStringsCapacity(position); stringBytes[position].clear(); stringBytes[position].put(bytes); - + stringBytes[position].put( (byte) 0); // add extra zero } catch (UnsupportedEncodingException e) { System.err.println("ERROR! UTF-8 not supported by the JVM!"); } @@ -356,17 +362,22 @@ public void setString(int index, String value) { byte[] bytes; try { - bytes = value.getBytes("UTF-8"); - + if (value == null) { + //set null values to an empty string + bytes = new String().getBytes("UTF-8"); + } else { + bytes = value.getBytes("UTF-8"); + } + // set length of string in header stringHeader.put(index, bytes.length); // make sure there is space for the string - ensureStringsCapacity(); + ensureStringsCapacity(index); stringBytes[index].clear(); stringBytes[index].put(bytes); - + stringBytes[index].put( (byte) 0); // add extra zero } catch (UnsupportedEncodingException e) { System.err.println("ERROR! UTF-8 not supported by the JVM!"); } @@ -434,7 +445,8 @@ public String getString(int index) throws IOException { } if (stringBytes.length <= index) { - throw new IOException("cannot get string at index " + index + " in call" + this + " header does not match content!"); + throw new IOException("cannot get string at index " + index + " in call" + this + + " header does not match content!"); } @@ -445,7 +457,7 @@ public String getString(int index) throws IOException { } byte[] bytes = new byte[utf8length]; stringBytes[index].position(0); - stringBytes[index].limit(utf8length); + stringBytes[index].limit(utf8length +1 ); // account for extra zero stringBytes[index].get(bytes); return new String(bytes, 0, utf8length, "UTF-8"); @@ -497,9 +509,9 @@ private void setStringLimitsFromHeader() throws IOException { } for (int i = 0; i < getStringCount(); i++) { - int utf8Length = stringHeader.get(i); + int utf8Length = stringHeader.get(i) +1; // account for extra zero - stringBytes[i].clear().limit(utf8Length); + stringBytes[i].clear().limit(utf8Length ); } // set the limit of the rest of the string bytes to 0 @@ -517,7 +529,21 @@ public void writeTo(SocketChannel channel) throws IOException { setStringLimitsFromHeader(); // write to channel - channel.write(byteBuffers); + // channel.write(byteBuffers); + + // write all bufferd to channel + boolean done = false; + + while(!done) { + channel.write(byteBuffers); + + done = true; + for (ByteBuffer buffer : byteBuffers) { + if (buffer.hasRemaining()) { + done = false; + } + } + } // alternative, debugging version of writing buffers // for (ByteBuffer buffer : byteBuffers) { @@ -627,7 +653,7 @@ public boolean ensureStringsCapacity() { } for (int i = 0; i < getStringCount(); i++) { - int stringLength = stringHeader.get(i); + int stringLength = stringHeader.get(i) +1 ; // account for extra zero if (stringBytes[i] == null || stringLength > stringBytes[i].capacity()) { stringBytes[i] = ByteBuffer.allocateDirect(stringLength); @@ -652,6 +678,49 @@ public boolean ensureStringsCapacity() { return buffersUpdated; } + public boolean ensureStringsCapacity(int index) { + // checking if the string header is big enough is checked above, so + // we + // only check if all strings listed in the header + boolean buffersUpdated = false; + + if (stringBytes.length < getStringCount()) { + ByteBuffer[] oldStringBytes = stringBytes; + stringBytes = new ByteBuffer[getStringCount()]; + for (int i = 0; i < oldStringBytes.length; i++) { + stringBytes[i] = oldStringBytes[i]; + } + buffersUpdated = true; + } + + if (buffersUpdated) { + // update byte buffers array + ByteBuffer[] newByteBuffers = new ByteBuffer[allButStringByteBuffers.length + stringBytes.length]; + for (int i = 0; i < allButStringByteBuffers.length; i++) { + newByteBuffers[i] = allButStringByteBuffers[i]; + } + for (int i = 0; i < stringBytes.length; i++) { + newByteBuffers[allButStringByteBuffers.length + i] = stringBytes[i]; + } + byteBuffers = newByteBuffers; + + + //System.err.println("ensureStringsCapacity() Updated buffers to " + Arrays.toString(byteBuffers)); + } + + { + int stringLength = stringHeader.get(index) +1; // account for extra zero + if (stringBytes[index] == null || stringLength > stringBytes[index].capacity()) { + + stringBytes[index] = ByteBuffer.allocateDirect(stringLength); + byteBuffers[allButStringByteBuffers.length + index] = stringBytes[index]; + } + } + + + return buffersUpdated; + } + public boolean readFrom(SocketChannel channel) throws IOException { boolean updatedBuffers = false; diff --git a/src/amuse/datamodel/grids.py b/src/amuse/datamodel/grids.py index a17692223..edf3b4043 100644 --- a/src/amuse/datamodel/grids.py +++ b/src/amuse/datamodel/grids.py @@ -318,16 +318,25 @@ def positions(indices, length, n): return result -def new_rectilinear_grid(shape, axes_cell_boundaries, axes_names = "xyz",offset=None): +def new_rectilinear_grid(shape, axes_cell_boundaries=None, cell_centers=None, axes_names = "xyz",offset=None): """Returns a rectilinear grid with cells at positions midway given cell boundaries. """ if len(axes_names) 0: sendbuffer = numpy.array(array, dtype='b') self.mpi_send(comm, [sendbuffer, MPI.C_BOOL or MPI.BYTE]) - - def string_lengths(self, array): - lengths = numpy.zeros(len(array), dtype='i') - index = 0 - - for string in array: - lengths[index] = len(string) - index += 1 - - return lengths - + def set_error(self, message): self.strings = [message] self.error = True @@ -2058,12 +2052,16 @@ def receive_strings(self, socket, count): if count > 0: lengths = self.receive_ints(socket, count) + total = lengths.sum() + len(lengths) + + data_bytes = self._receive_all(total, socket) + strings = [] - - for i in range(count): - data_bytes = self._receive_all(lengths[i], socket) - strings.append(str(data_bytes.decode('utf-8'))) - + begin = 0 + for size in lengths: + strings.append(data_bytes[begin:begin + size].decode('utf-8')) + begin = begin + size + 1 + return strings else: return [] @@ -2122,21 +2120,16 @@ def send_floats(self, socket, array): socket.sendall(data_buffer.tostring()) def send_strings(self, socket, array): - header = [] - data_bytes = [] - - for i in range(len(array)): + if len(array) > 0: - #logger.debug("sending string %s", array[i]) + lengths = numpy.array( [len(s) for s in array] ,dtype='int32') + chars=(chr(0).join(array)+chr(0)).encode("utf-8") - utf8_string = array[i].encode('utf-8') - header.append(len(utf8_string)) - data_bytes.append(utf8_string) - - self.send_ints(socket, header); - - for i in range(len(data_bytes)): - socket.sendall(data_bytes[i]) + if len(chars) != lengths.sum()+len(lengths): + raise Exception("send_strings size mismatch {0} vs {1}".format( len(chars) , lengths.sum()+len(lengths) )) + + self.send_ints(socket, lengths); + socket.sendall(chars) def send_booleans(self, socket, array): if len(array) > 0: diff --git a/src/amuse/rfi/core.py b/src/amuse/rfi/core.py index a02a78971..c6182c261 100644 --- a/src/amuse/rfi/core.py +++ b/src/amuse/rfi/core.py @@ -12,7 +12,7 @@ import inspect import functools -#from collections import OrderedDict +# from collections import OrderedDict from subprocess import Popen, PIPE diff --git a/src/amuse/rfi/tools/create_c.py b/src/amuse/rfi/tools/create_c.py index 9f75772a6..d58ecbbb7 100644 --- a/src/amuse/rfi/tools/create_c.py +++ b/src/amuse/rfi/tools/create_c.py @@ -648,12 +648,21 @@ if(header_in[HEADER_STRING_COUNT] > 0) { receive_array_sockets(string_sizes_in, header_in[HEADER_STRING_COUNT] * sizeof(int), socketfd, rank); MPI_Bcast(string_sizes_in, header_in[HEADER_STRING_COUNT], MPI_INT, 0, MPI_COMM_WORLD); - for (int i = 0; i < header_in[HEADER_STRING_COUNT]; i++) { - strings_in[i] = new char[string_sizes_in[i] + 1]; - receive_array_sockets(strings_in[i], string_sizes_in[i], socketfd, rank); - MPI_Bcast(strings_in[i], string_sizes_in[i], MPI_CHARACTER, 0, MPI_COMM_WORLD); - strings_in[i][string_sizes_in[i]] = '\\0'; + + int total_string_size = 0; + for (int i = 0; i < header_in[HEADER_STRING_COUNT];i++) { + total_string_size += string_sizes_in[i] + 1; } + + characters_in = new char[total_string_size]; + receive_array_sockets(characters_in, total_string_size, socketfd, rank); + MPI_Bcast(characters_in, total_string_size, MPI_CHARACTER, 0, MPI_COMM_WORLD); + + int offset = 0; + for (int i = 0 ; i < header_in[HEADER_STRING_COUNT];i++) { + strings_in[i] = characters_in + offset; + offset += string_sizes_in[i] + 1; + } } header_out[HEADER_FLAGS] = 0; @@ -701,19 +710,36 @@ } if(header_out[HEADER_STRING_COUNT] > 0) { - for (int i = 0; i < header_out[HEADER_STRING_COUNT]; i++) { - string_sizes_out[i] = strlen(strings_out[i]); + int offset = 0; + for( int i = 0; i < header_out[HEADER_STRING_COUNT] ; i++) { + int length = strlen(strings_out[i]); + string_sizes_out[i] = length; + offset += length + 1; } - send_array_sockets(string_sizes_out, header_out[HEADER_STRING_COUNT] * sizeof(int), socketfd, 0); - - for (int i = 0; i < header_out[HEADER_STRING_COUNT]; i++) { - send_array_sockets(strings_out[i], string_sizes_out[i] * sizeof(char), socketfd, 0); + + characters_out = new char[offset + 1]; + offset = 0; + + for( int i = 0; i < header_out[HEADER_STRING_COUNT] ; i++) { + strcpy(characters_out+offset, strings_out[i]); + offset += string_sizes_out[i] + 1; } + + send_array_sockets(string_sizes_out, header_out[HEADER_STRING_COUNT] * sizeof(int), socketfd, 0); + send_array_sockets(characters_out, offset * sizeof(char), socketfd, 0); } //fprintf(stderr, "sockets_mpicall done\\n"); } - + if (characters_in) { + delete[] characters_in; + characters_in = 0; + } + + if (characters_out) { + delete[] characters_out; + characters_out = 0; + } } delete_arrays(); @@ -827,11 +853,20 @@ if(header_in[HEADER_STRING_COUNT] > 0) { receive_array_sockets(string_sizes_in, header_in[HEADER_STRING_COUNT] * sizeof(int), socketfd, 0); - for (int i = 0; i < header_in[HEADER_STRING_COUNT]; i++) { - strings_in[i] = new char[string_sizes_in[i] + 1]; - receive_array_sockets(strings_in[i], string_sizes_in[i], socketfd, 0); - strings_in[i][string_sizes_in[i]] = '\\0'; + + int total_string_size = 0; + for (int i = 0; i < header_in[HEADER_STRING_COUNT];i++) { + total_string_size += string_sizes_in[i] + 1; } + + characters_in = new char[total_string_size]; + receive_array_sockets(characters_in, total_string_size, socketfd, 0); + + int offset = 0; + for (int i = 0 ; i < header_in[HEADER_STRING_COUNT];i++) { + strings_in[i] = characters_in + offset; + offset += string_sizes_in[i] + 1; + } } header_out[HEADER_FLAGS] = 0; @@ -876,16 +911,34 @@ } if(header_out[HEADER_STRING_COUNT] > 0) { - for (int i = 0; i < header_out[HEADER_STRING_COUNT]; i++) { - string_sizes_out[i] = strlen(strings_out[i]); - } - send_array_sockets(string_sizes_out, header_out[HEADER_STRING_COUNT] * sizeof(int), socketfd, 0); - - for (int i = 0; i < header_out[HEADER_STRING_COUNT]; i++) { - send_array_sockets(strings_out[i], string_sizes_out[i] * sizeof(char), socketfd, 0); - } + int offset = 0; + for( int i = 0; i < header_out[HEADER_STRING_COUNT] ; i++) { + int length = strlen(strings_out[i]); + string_sizes_out[i] = length; + offset += length + 1; + } + + characters_out = new char[offset + 1]; + offset = 0; + + for( int i = 0; i < header_out[HEADER_STRING_COUNT] ; i++) { + strcpy(characters_out+offset, strings_out[i]); + offset += string_sizes_out[i] + 1; + } + + send_array_sockets(string_sizes_out, header_out[HEADER_STRING_COUNT] * sizeof(int), socketfd, 0); + send_array_sockets(characters_out, offset * sizeof(char), socketfd, 0); } + if (characters_in) { + delete[] characters_in; + characters_in = 0; + } + + if (characters_out) { + delete[] characters_out; + characters_out = 0; + } //fprintf(stderr, "call done\\n"); } delete_arrays(); diff --git a/src/amuse/rfi/tools/create_fortran.py b/src/amuse/rfi/tools/create_fortran.py index e71b6edc0..a47259a51 100644 --- a/src/amuse/rfi/tools/create_fortran.py +++ b/src/amuse/rfi/tools/create_fortran.py @@ -88,9 +88,12 @@ character (c_char), allocatable, target :: strings_in(:) * 256 character (c_char), allocatable, target :: strings_out(:) * 256 + + character (len=1000000) :: characters_in + character (len=1000000) :: characters_out - character (kind=c_char, len=1000000), target :: characters_in - character (kind=c_char, len=1000000), target :: characters_out + character (kind=c_char), target :: c_characters_in(1000000) + character (kind=c_char), target :: c_characters_out(1000000) """ MODULE_GLOBALS_STRING = """ @@ -435,6 +438,16 @@ maximum_size = string_sizes_in(i) end if end do + + if(maximum_size.GT.256) then + print*, "fortran_worker reports too large string" + stop + endif + + if(total_string_length.GT.1000000) then + print*, "fortran_worker reports too large string message" + stop + endif call MPI_BCast(characters_in, total_string_length, MPI_CHARACTER, 0, parent, ioError); @@ -446,7 +459,6 @@ offset = offset + string_sizes_in(i) + 1 !print*, 'fortran: strings_in(i) ', i, strings_in(i) , ' of length ', string_sizes_in(i), & !' actually of size ', len_trim(strings_in(i)) - end do end if @@ -496,6 +508,12 @@ offset = offset + string_sizes_out(i) + 1 characters_out(offset-1:offset-1) = char(0) end do + + total_string_length=offset-1 + if(total_string_length.GT.1000000) then + print*, "fortran_worker reports too large string message" + stop + endif call MPI_SEND(string_sizes_out, header_out(HEADER_STRING_COUNT), MPI_INTEGER, 0, 999, parent, ioerror) call MPI_SEND(characters_out, offset -1, MPI_CHARACTER, 0, 999, parent, ioerror) @@ -534,8 +552,8 @@ implicit none integer :: max_call_count = 255 - integer :: must_run_loop - integer i, call_count, port + integer :: must_run_loop, maximum_size, total_string_length + integer :: i, offset, call_count, port character(len=32) :: port_string character(kind=c_char, len=64) :: host logical (c_bool), allocatable, target :: c_booleans_in(:) @@ -639,11 +657,43 @@ strings_in = ' ' call receive_integers(c_loc(string_sizes_in), header_in(HEADER_STRING_COUNT)) + maximum_size = 0 + total_string_length = 0 do i = 1, header_in(HEADER_STRING_COUNT), 1 - strings_in(i) = ' ' - call receive_string(c_loc(strings_in(i)), string_sizes_in(i)) + total_string_length = total_string_length + string_sizes_in(i) + 1 + if (string_sizes_in(i) .gt. maximum_size) then + maximum_size = string_sizes_in(i) + end if end do + + if(maximum_size.GT.256) then + print*, "fortran_worker reports too large string" + stop + endif + + if(total_string_length.GT.1000000) then + print*, "fortran_worker reports too large string message" + stop + endif + call receive_string(c_loc(c_characters_in), total_string_length) + + ! this trick is necessary on older gfortran compilers (~<4.9) + ! as c_loc needs character(len=1) + do i=1, total_string_length + characters_in(i:i)=c_characters_in(i) + enddo + + offset = 1 + do i = 1, header_in(HEADER_STRING_COUNT), 1 + strings_in(i) = ' ' + strings_in(i) = characters_in(offset : (offset + string_sizes_in(i))) + strings_in(i)((string_sizes_in(i) + 1):(string_sizes_in(i) + 1)) = ' ' + offset = offset + string_sizes_in(i) + 1 + !print*, 'fortran: strings_in(i) ', i, strings_in(i) , ' of length ', string_sizes_in(i), & + !' actually of size ', len_trim(strings_in(i)) + end do + end if header_out = 0 @@ -680,19 +730,32 @@ end if if (header_out(HEADER_STRING_COUNT) .gt. 0) then - + offset = 1 do i = 1, header_out(HEADER_STRING_COUNT),1 + string_sizes_out(i) = len_trim(strings_out(i)) - - !print*, 'fortran: sending strings, strings_out(i) ', i, strings_out(i) , ' of length ', string_sizes_out(i), & - !'actually of size ', len_trim(strings_out(i)) + + !print*, 'fortran: sending strings, strings_out(i) ', i, strings_out(i) , ' of length ', string_sizes_out(i), & + !' actually of size ', len_trim(strings_out(i)) + + characters_out(offset:offset+string_sizes_out(i)) = strings_out(i) + offset = offset + string_sizes_out(i) + 1 + characters_out(offset-1:offset-1) = char(0) end do - - call send_integers(c_loc(string_sizes_out), header_out(HEADER_STRING_COUNT)) + + total_string_length=offset-1 + + if(total_string_length.GT.1000000) then + print*, "fortran_worker reports too large string message" + stop + endif - do i = 1, header_out(HEADER_STRING_COUNT),1 - call send_string(c_loc(strings_out(i)), string_sizes_out(i)) - end do + do i=1, total_string_length + c_characters_out(i)=characters_out(i:i) + enddo + + call send_integers(c_loc(string_sizes_out), header_out(HEADER_STRING_COUNT)) + call send_string(c_loc(c_characters_out), offset-1 ) end if end do @@ -738,8 +801,8 @@ integer :: provided integer :: max_call_count = 255 - integer :: must_run_loop - integer i, call_count, port, rank, ioerror + integer :: must_run_loop, maximum_size, total_string_length + integer :: i, offset, call_count, port, rank, ioerror character(len=32) :: port_string character(kind=c_char, len=64) :: host logical (c_bool), allocatable, target :: c_booleans_in(:) @@ -867,22 +930,52 @@ end if if (header_in(HEADER_STRING_COUNT) .gt. 0) then - strings_in = ' ' - + if (rank .eq. 0) then call receive_integers(c_loc(string_sizes_in), header_in(HEADER_STRING_COUNT)) end if - call MPI_BCast(string_sizes_in, header_in(HEADER_STRING_COUNT), MPI_INTEGER, 0, MPI_COMM_WORLD, ioError); + maximum_size = 0 + total_string_length = 0 do i = 1, header_in(HEADER_STRING_COUNT), 1 - strings_in(i) = ' ' - if (rank .eq. 0) then - call receive_string(c_loc(strings_in(i)), string_sizes_in(i)) - end if - call MPI_BCast(strings_in(i), string_sizes_in(i), MPI_CHARACTER, 0, MPI_COMM_WORLD, ioError); + total_string_length = total_string_length + string_sizes_in(i) + 1 + if (string_sizes_in(i) .gt. maximum_size) then + maximum_size = string_sizes_in(i) + end if end do + + if(maximum_size.GT.256) then + print*, "fortran_worker reports too large string" + stop + endif + + if(total_string_length.GT.1000000) then + print*, "fortran_worker reports too large string message" + stop + endif + + if (rank .eq. 0) then + call receive_string(c_loc(c_characters_in), total_string_length) + endif + + do i=1, total_string_length + characters_in(i:i)=c_characters_in(i) + enddo + + call MPI_BCast(characters_in, total_string_length, MPI_CHARACTER, 0, MPI_COMM_WORLD, ioError); + + offset = 1 + do i = 1, header_in(HEADER_STRING_COUNT), 1 + strings_in(i) = ' ' + strings_in(i) = characters_in(offset : (offset + string_sizes_in(i))) + strings_in(i)((string_sizes_in(i) + 1):(string_sizes_in(i) + 1)) = ' ' + offset = offset + string_sizes_in(i) + 1 + !print*, 'fortran: strings_in(i) ', i, strings_in(i) , ' of length ', string_sizes_in(i), & + !' actually of size ', len_trim(strings_in(i)) + end do + end if header_out = 0 @@ -924,20 +1017,33 @@ end if if (header_out(HEADER_STRING_COUNT) .gt. 0) then - + offset = 1 do i = 1, header_out(HEADER_STRING_COUNT),1 + string_sizes_out(i) = len_trim(strings_out(i)) - - !print*, 'fortran: sending strings, strings_out(i) ', i, strings_out(i) , ' of length ', string_sizes_out(i), & - !'actually of size ', len_trim(strings_out(i)) + + !print*, 'fortran: sending strings, strings_out(i) ', i, strings_out(i) , ' of length ', string_sizes_out(i), & + !' actually of size ', len_trim(strings_out(i)) + + characters_out(offset:offset+string_sizes_out(i)) = strings_out(i) + offset = offset + string_sizes_out(i) + 1 + characters_out(offset-1:offset-1) = char(0) end do - + + total_string_length=offset-1 + + if(total_string_length.GT.1000000) then + print*, "fortran_Worker reports too large string message" + stop + endif + + do i=1, total_string_length + c_characters_out(i)=characters_out(i:i) + enddo + call send_integers(c_loc(string_sizes_out), header_out(HEADER_STRING_COUNT)) - - do i = 1, header_out(HEADER_STRING_COUNT),1 - call send_string(c_loc(strings_out(i)), string_sizes_out(i)) - end do - end if + call send_string(c_loc(c_characters_out), offset-1 ) + end if end if end do diff --git a/src/amuse/rfi/tools/create_java.py b/src/amuse/rfi/tools/create_java.py index 56faaf9b3..29ff6ebb8 100644 --- a/src/amuse/rfi/tools/create_java.py +++ b/src/amuse/rfi/tools/create_java.py @@ -286,10 +286,11 @@ stringHeader.put(0, bytes.length); - ensureStringsCapacity(); + ensureStringsCapacity(0); stringBytes[0].clear(); stringBytes[0].put(bytes); + stringBytes[0].put( (byte) 0); // add extra zero } catch (UnsupportedEncodingException e) { System.err.println("could not set error: " + e); stringHeader.put(0, 0); @@ -330,10 +331,11 @@ stringHeader.put(position, bytes.length); // make sure there is space for the string - ensureStringsCapacity(); + ensureStringsCapacity(position); stringBytes[position].clear(); stringBytes[position].put(bytes); + stringBytes[position].put( (byte) 0); // add extra zero } catch (UnsupportedEncodingException e) { System.err.println("ERROR! UTF-8 not supported by the JVM!"); @@ -356,10 +358,11 @@ stringHeader.put(index, bytes.length); // make sure there is space for the string - ensureStringsCapacity(); + ensureStringsCapacity(index); stringBytes[index].clear(); stringBytes[index].put(bytes); + stringBytes[index].put( (byte) 0); // add extra zero } catch (UnsupportedEncodingException e) { System.err.println("ERROR! UTF-8 not supported by the JVM!"); @@ -431,7 +434,7 @@ } byte[] bytes = new byte[utf8length]; stringBytes[index].position(0); - stringBytes[index].limit(utf8length); + stringBytes[index].limit(utf8length+1); // account for extra zero stringBytes[index].get(bytes); return new String(bytes, 0, utf8length, "UTF-8"); @@ -488,7 +491,7 @@ for (int i = 0; i < getStringCount(); i++) { int utf8Length = stringHeader.get(i); - stringBytes[i].clear().limit(utf8Length); + stringBytes[i].clear().limit(utf8Length + 1); // account for extra zero } // set the limit of the rest of the string bytes to 0 @@ -611,7 +614,7 @@ } for (int i = 0; i < getStringCount(); i++) { - int stringLength = stringHeader.get(i); + int stringLength = stringHeader.get(i) +1; // account for extra zero if (stringBytes[i] == null || stringLength > stringBytes[i].capacity()) { stringBytes[i] = ByteBuffer.allocateDirect(stringLength); @@ -637,6 +640,49 @@ return buffersUpdated; } + public boolean ensureStringsCapacity(int index) { + // checking if the string header is big enough is checked above, so + // we + // only check if all strings listed in the header + boolean buffersUpdated = false; + + if (stringBytes.length < getStringCount()) { + ByteBuffer[] oldStringBytes = stringBytes; + stringBytes = new ByteBuffer[getStringCount()]; + for (int i = 0; i < oldStringBytes.length; i++) { + stringBytes[i] = oldStringBytes[i]; + } + buffersUpdated = true; + } + + if (buffersUpdated) { + // update byte buffers array + ByteBuffer[] newByteBuffers = new ByteBuffer[allButStringByteBuffers.length + stringBytes.length]; + for (int i = 0; i < allButStringByteBuffers.length; i++) { + newByteBuffers[i] = allButStringByteBuffers[i]; + } + for (int i = 0; i < stringBytes.length; i++) { + newByteBuffers[allButStringByteBuffers.length + i] = stringBytes[i]; + } + byteBuffers = newByteBuffers; + + + //System.err.println("ensureStringsCapacity() Updated buffers to " + Arrays.toString(byteBuffers)); + } + + { + int stringLength = stringHeader.get(index) +1; // account for extra zero + if (stringBytes[index] == null || stringLength > stringBytes[index].capacity()) { + + stringBytes[index] = ByteBuffer.allocateDirect(stringLength); + byteBuffers[allButStringByteBuffers.length + index] = stringBytes[index]; + } + } + + + return buffersUpdated; + } + boolean readFrom(SocketChannel channel) throws IOException { boolean updatedBuffers = false; diff --git a/src/amuse/support/interface.py b/src/amuse/support/interface.py index 3ee1fbece..206f72f7e 100644 --- a/src/amuse/support/interface.py +++ b/src/amuse/support/interface.py @@ -22,6 +22,7 @@ from amuse.datamodel import incode_storage import itertools +from collections import defaultdict class ConvertArgumentsException(core.IncompatibleUnitsException): formatstring = "{0}" @@ -859,22 +860,25 @@ class HandleParameters(HandleCodeInterfaceAttributeAccess): def __init__(self, interface): self.property_definitions = {} self.interface = interface - self.definitions = [] - self.parameters = None + self.definitions = defaultdict(list,parameters=[]) + self.parameters = {} def supports(self, name, was_found): - return name == 'parameters' + return name in self.definitions.keys() def get_attribute(self, name, value): if not self.parameters: - self.parameters = parameters.new_parameters_instance_with_docs(self.definitions, self.interface) + for n,d in self.definitions.iteritems(): + self.parameters[n] = parameters.new_parameters_instance_with_docs(d, self.interface) - return self.parameters + return self.parameters[name or 'parameters'] def attribute_names(self): - return set(['parameters']) + return set(self.definitions.keys()) - def add_method_parameter(self, get_method, set_method, name, description, default_value = None,must_set_before_get = False, is_vector = False): + def add_method_parameter(self, get_method, set_method, name, description, + default_value = None,must_set_before_get = False, is_vector = False, + parameter_set='parameters'): if is_vector: definition = parameters.ModuleVectorMethodParameterDefinition( get_method, @@ -893,18 +897,19 @@ def add_method_parameter(self, get_method, set_method, name, description, defaul default_value, must_set_before_get = must_set_before_get ) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) - def add_alias_parameter(self, name, aliased_name, description): + def add_alias_parameter(self, name, aliased_name, description,parameter_set='parameters',alias_set=None): definition = parameters.AliasParameterDefinition( name, aliased_name, - description + description, + alias_set=alias_set ) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) - def add_caching_parameter(self, function_name, parameter_name, name, description, default_value = None): + def add_caching_parameter(self, function_name, parameter_name, name, description, default_value = None,parameter_set='parameters'): definition = parameters.ModuleCachingParameterDefinition( function_name, parameter_name, @@ -912,9 +917,9 @@ def add_caching_parameter(self, function_name, parameter_name, name, description description, default_value ) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) - def add_boolean_parameter(self, get_method, set_method, name, description, default_value = None): + def add_boolean_parameter(self, get_method, set_method, name, description, default_value = None,parameter_set='parameters'): definition = parameters.ModuleBooleanParameterDefinition( get_method, set_method, @@ -922,15 +927,15 @@ def add_boolean_parameter(self, get_method, set_method, name, description, defau description, default_value ) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) - def add_default_form_parameter(self,name,description,default): + def add_default_form_parameter(self,name,description,default,parameter_set='parameters'): if isinstance(default,bool): - self.add_boolean_parameter("get_"+name,"set_"+name,name,description,default) + self.add_boolean_parameter("get_"+name,"set_"+name,name,description,default,parameter_set='parameters') else: - self.add_method_parameter("get_"+name,"set_"+name,name,description,default) + self.add_method_parameter("get_"+name,"set_"+name,name,description,default,parameter_set='parameters') - def add_array_parameter(self, get_method, set_method, range_method, name, description): + def add_array_parameter(self, get_method, set_method, range_method, name, description,parameter_set='parameters'): definition = parameters.ModuleArrayParameterDefinition( get_method, set_method, @@ -938,7 +943,7 @@ def add_array_parameter(self, get_method, set_method, range_method, name, descri name, description ) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) def has_name(self, name): return name == 'PARAMETER' @@ -948,10 +953,10 @@ def setup(self, object): - def add_vector_parameter(self, name, description, parameter_names): + def add_vector_parameter(self, name, description, parameter_names,parameter_set='parameters'): default_value = None for parameter_name in parameter_names: - for defined_parameter in self.definitions: + for defined_parameter in self.definitions[parameter_set]: if defined_parameter.name == parameter_name: if default_value is None: if not is_quantity(defined_parameter.default_value): @@ -965,11 +970,11 @@ def add_vector_parameter(self, name, description, parameter_names): parameter_names, default_value ) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) - def add_interface_parameter(self, name, description, default_value,state_guard=None): + def add_interface_parameter(self, name, description, default_value,state_guard=None,parameter_set='parameters'): definition=parameters.InterfaceParameterDefinition(name,description,default_value,state_guard=state_guard) - self.definitions.append(definition) + self.definitions[parameter_set].append(definition) class HandleErrorCodes(HandleCodeInterfaceAttributeAccess): def __init__(self, interface): @@ -1521,6 +1526,11 @@ def __setstate__(self, state): def data_store_names(self): self.before_get_data_store_names() return self.get_handler('PARTICLES').mapping_from_name_to_set_definition.keys() + + def parameter_set_names(self): + #~ self.before_get_data_store_names() + return self.get_handler('PARAMETER').definitions.keys() + class IncorrectMethodDefinition(IncorrectWrappedMethodException): formatstring = "Incorrect definition of method '{0}' of class '{1}', the number of {4} do not match, expected {2}, actual {3}." diff --git a/src/amuse/support/parameter_tools.py b/src/amuse/support/parameter_tools.py new file mode 100644 index 000000000..254a81505 --- /dev/null +++ b/src/amuse/support/parameter_tools.py @@ -0,0 +1,65 @@ +try: + import f90nml + HAS_F90NML=True +except: + HAS_F90NML=False +from collections import defaultdict +from amuse.units.quantities import new_quantity, to_quantity, is_quantity + + +# CodeWithNamelistParameters +# +# namelist_parameters=( +# dict(name="name", group_name="name", short="codename", dtype="int32", default=64, description="description", ptype="nml" [, set_name="name"]), ... +# ) + +class CodeWithNamelistParameters(object): + def __init__(self, namelist_parameters): + self._namelist_parameters=dict([((x["short"],x["group_name"]),x) for x in namelist_parameters]) + + def define_parameters(self,object): + for p in self._namelist_parameters.values(): + if p["ptype"] in ["nml", "nml+normal"]: + parameter_set_name=p.get("set_name", "parameters_"+p["group_name"]) + object.add_interface_parameter( p["name"], p["description"], p["default"], "before_set_interface_parameter", parameter_set=parameter_set_name) + + def read_namelist_parameters(self, inputfile): + + self._nml_file=inputfile + self._nml_params = f90nml.read(inputfile) + + for group, d in self._nml_params.iteritems(): + for short, val in d.iteritems(): + key=(short,group.upper()) + if key in self._namelist_parameters: + group_name=self._namelist_parameters[key]["group_name"] + name=self._namelist_parameters[key]["name"] + parameter_set_name=self._namelist_parameters[key].get("set_name", "parameters_"+group_name) + parameter_set=getattr(self, parameter_set_name) + if is_quantity(self._namelist_parameters[key]["default"]): + setattr(parameter_set, name, new_quantity(val, to_quantity(self._namelist_parameters[key]["default"]).unit) ) + else: + setattr(parameter_set, name, val ) + else: + print "'%s' of group '%s' not in the namelist_parameters"%(short, group) + + def write_namelist_parameters(self, outputfile, do_patch=False, nml_file=None): + patch=defaultdict( dict ) + for p in self._namelist_parameters.values(): + name=p["name"] + group_name=p["group_name"] + group=patch[group_name] + short=p["short"] + parameter_set_name=p.get("set_name", "parameters_"+group_name) + parameter_set=getattr(self, parameter_set_name) + if getattr(parameter_set, name) is None: # omit if value is None + continue + if is_quantity(p["default"]): + group[short]=to_quantity(getattr(parameter_set, name)).value_in(p["default"].unit) + else: + group[short]=getattr(parameter_set, name) + + if do_patch: + f90nml.patch(nml_file or self._nml_file,patch,outputfile) + else: + f90nml.write(patch, outputfile, force=True) diff --git a/src/amuse/test/compile_tools.py b/src/amuse/test/compile_tools.py index 258f81782..66b549d4e 100644 --- a/src/amuse/test/compile_tools.py +++ b/src/amuse/test/compile_tools.py @@ -7,6 +7,7 @@ from amuse.rfi.tools import create_java from amuse.rfi.tools import create_c +from amuse.rfi.tools import create_fortran from amuse.rfi.core import config from amuse.test.amusetest import get_amuse_root_dir @@ -193,7 +194,7 @@ def c_build(exename, objectnames, extra_args=[]): def build_worker(codestring, path_to_results, specification_class, write_header=True, - extra_args=[]): + extra_args=[], needs_mpi = False): path = os.path.abspath(path_to_results) codefile = os.path.join(path, "code.o") headerfile = os.path.join(path, "worker_code.h") @@ -204,7 +205,7 @@ def build_worker(codestring, path_to_results, specification_class, write_header= uc = create_c.GenerateACHeaderStringFromASpecificationClass() uc.specification_class = specification_class - uc.needs_mpi = False + uc.needs_mpi = needs_mpi header = uc.result if write_header: @@ -214,7 +215,7 @@ def build_worker(codestring, path_to_results, specification_class, write_header= uc = create_c.GenerateACSourcecodeStringFromASpecificationClass() uc.specification_class = specification_class - uc.needs_mpi = False + uc.needs_mpi = needs_mpi code = uc.result cxx_compile(interfacefile, code if write_header else header+"\n\n"+code, @@ -362,7 +363,7 @@ def fortran_compile(objectname, string, extra_args=[]): raise Exception("Could not compile {0}\nstdout:\n{1}\nstderr:\n{2}\narguments:\n{3}".format(objectname, stdout, stderr, ' '.join(arguments))) -def fortran_build(exename, objectnames): +def fortran_build(exename, objectnames, extra_args=[]): if os.path.exists(exename): os.remove(exename) @@ -372,6 +373,7 @@ def fortran_build(exename, objectnames): arguments.append("-L{0}/lib/forsockets".format(get_amuse_root_dir())) arguments.append("-Wall") arguments.append("-lforsockets") + arguments.extend(extra_args) arguments.append("-o") arguments.append(exename) @@ -386,39 +388,6 @@ def fortran_build(exename, objectnames): if process.returncode != 0 or not os.path.exists(exename): raise Exception("Could not build {0}\nstdout:\n{1}\nstderr:\n{2}\narguments:\n{3}".format(exename, stdout, stderr, ' '.join(arguments))) - -def f90_compile(objectname, string, mpidir): - root, ext = os.path.splitext(objectname) - sourcename = root + '.f90' - if os.path.exists(objectname): - os.remove(objectname) - with open(sourcename, "w") as f: - f.write(string) - - rootdir = get_amuse_root_dir() - arguments = get_mpif90_arguments() - arguments.extend(["-I", "{0}/lib/stopcond".format(rootdir, mpidir), "-I", rootdir + "/lib/forsockets", "-c", "-o", objectname, sourcename]) - process, stderr, stdout = open_subprocess(arguments) - if not os.path.exists(objectname): # or process.poll() == 1: - raise Exception("Could not compile {0}\nstdout:\n{1}\nstderr:\n{2}\narguments:\n{3}".format(objectname, stdout, stderr, ' '.join(arguments))) - - -def f90_build(exename, objectnames, libname): - rootdir = get_amuse_root_dir() - - arguments = get_mpif90_arguments() - arguments.extend(objectnames) - arguments.append("-o") - arguments.append(exename) - arguments.extend(["-L{0}/lib/stopcond".format(rootdir), "-l"+libname]) - arguments.extend(["-L" + rootdir + "/lib/forsockets","-lforsockets"]) - arguments.extend(get_ld_flags().split()) - print 'build command:' - print ' '.join(arguments) - process, stderr, stdout = open_subprocess(arguments) - if process.returncode != 0: - raise Exception("Could not build {0}\nstdout:\n{1}\nstderr:\n{2}\narguments:\n{3}".format(exename, stdout, stderr, ' '.join(arguments))) - def build_java_worker(codestring, path_to_results, specification_class): path = os.path.abspath(path_to_results) exefile = os.path.join(path,"java_worker") @@ -499,3 +468,21 @@ def build_java_worker(codestring, path_to_results, specification_class): os.chmod(exefile, 0777) return exefile + +def build_fortran_worker(codestring, path_to_results, interface, needs_mpi = True, + extra_fflags=[], extra_ldflags=[]): + + path = os.path.abspath(path_to_results) + codefile = os.path.join(path,"code.o") + interfacefile = os.path.join(path,"interface.o") + exefile = os.path.join(path,"fortran_worker") + + fortran_compile(codefile, codestring, extra_args=extra_fflags) + + uc = create_fortran.GenerateAFortranSourcecodeStringFromASpecificationClass() + uc.specification_class = interface + uc.needs_mpi = needs_mpi + string = uc.result + fortran_compile(interfacefile, string, extra_args=extra_fflags) + fortran_build(exefile, [interfacefile, codefile], extra_args=extra_ldflags ) + return exefile diff --git a/test/compile_tests/test_c_implementation.py b/test/compile_tests/test_c_implementation.py index 0174f8546..2ebe8cb96 100644 --- a/test/compile_tests/test_c_implementation.py +++ b/test/compile_tests/test_c_implementation.py @@ -432,6 +432,36 @@ def test7(self): self.assertEquals(out[0], "abc") self.assertEquals(out[1], "def") + def test7b(self): + instance = ForTestingInterface(self.exefile) + + out, error = instance.echo_int(numpy.arange(2000000)) + + self.assertEquals(error, [0]*2000000) + self.assertEquals(out, numpy.arange(2000000)) + + instance.stop() + + def test7c(self): + instance = ForTestingInterface(self.exefile) + + out, error = instance.echo_string(["abc"]*10) + + self.assertEquals(error, [0]*10) + self.assertEquals(out, ["abc"]*10) + + instance.stop() + + def test7d(self): + instance = ForTestingInterface(self.exefile) + + out, error = instance.echo_string(["abc"]*100000) + + self.assertEquals(error, [0]*100000) + self.assertEquals(out, ["abc"]*100000) + + instance.stop() + def test8(self): instance = ForTestingInterface(self.exefile) out1, out2, error = instance.echo_strings("abc","def") diff --git a/test/compile_tests/test_c_implementation_simplified.py b/test/compile_tests/test_c_implementation_simplified.py index 57e2bc48d..2985fd02b 100644 --- a/test/compile_tests/test_c_implementation_simplified.py +++ b/test/compile_tests/test_c_implementation_simplified.py @@ -242,35 +242,13 @@ def define_methods(self, object): class TestCImplementationInterface(TestWithMPI): - def build_worker(self): - path = os.path.abspath(self.get_path_to_results()) - codefile = os.path.join(path, "code.o") - interfacefile = os.path.join(path, "interface.o") - self.exefile = os.path.join(path, "c_worker") - - compile_tools.c_compile(codefile, codestring) - - uc = create_c.GenerateACHeaderStringFromASpecificationClass() - uc.specification_class = ForTestingInterface - uc.needs_mpi = False - header = uc.result - - uc = create_c.GenerateACSourcecodeStringFromASpecificationClass() - uc.specification_class = ForTestingInterface - uc.needs_mpi = False - code = uc.result - - string = '\n\n'.join([header, code]) - - compile_tools.cxx_compile(interfacefile, string) - compile_tools.c_build(self.exefile, [interfacefile, codefile]) - def setUp(self): super(TestCImplementationInterface, self).setUp() print "building...", self.check_can_compile_modules() try: - self.build_worker() + self.exefile = compile_tools.build_worker(codestring, self.get_path_to_results(), + ForTestingInterface, write_header=False) except Exception as ex: print ex raise diff --git a/test/compile_tests/test_cython_implementation.py b/test/compile_tests/test_cython_implementation.py index 452b2034c..ca998e4b5 100644 --- a/test/compile_tests/test_cython_implementation.py +++ b/test/compile_tests/test_cython_implementation.py @@ -215,7 +215,12 @@ def build_worker(self): os.chmod(self.exefile, 0777) - process, stdout, stderr = compile_tools.open_subprocess([config.compilers.cython, sourcename, '-o', cname]) + import mpi4py + process, stdout, stderr = compile_tools.open_subprocess([config.compilers.cython, + '-I', + mpi4py.get_include(), + sourcename, '-o', cname]) + if process.returncode == 0: compile_tools.wait_for_file(cname) diff --git a/test/compile_tests/test_fortran_implementation.py b/test/compile_tests/test_fortran_implementation.py index e7396472a..3c13e6a73 100644 --- a/test/compile_tests/test_fortran_implementation.py +++ b/test/compile_tests/test_fortran_implementation.py @@ -10,7 +10,6 @@ from amuse.units import nbody_system from amuse.units import units from amuse import datamodel -from amuse.rfi.tools import create_fortran from amuse.rfi import channel from amuse.rfi.core import * @@ -79,7 +78,7 @@ function hello_string(string_out) implicit none - character(len=4096) :: string_out + character(len=*) :: string_out integer :: hello_string string_out = 'hello' @@ -157,6 +156,12 @@ echo_logical = 0 end function +function get_element_status(ind,x) result(ret) + integer :: ind,ret + character(len=*) :: x + x="dry" + ret=0 +end function """ class ForTestingInterface(CodeInterface): @@ -286,6 +291,15 @@ def print_error_string(): function.result_type = 'int32' function.can_handle_array = True return function + + @legacy_function + def get_element_status(): + function = LegacyFunctionSpecification() + function.addParameter('index', dtype='i', direction=function.IN) + function.addParameter('status', dtype='string', direction=function.OUT) + function.result_type = 'int32' + function.can_handle_array = True + return function class ForTesting(InCodeComponentImplementation): @@ -317,28 +331,12 @@ def skip_if_fortran_does_not_support_mpi(self): return else: self.skip("cannot run test as fortran does not support iso c bindings") - - def build_worker(self): - - path = os.path.abspath(self.get_path_to_results()) - codefile = os.path.join(path,"code.o") - interfacefile = os.path.join(path,"interface.o") - self.exefile = os.path.join(path,"fortran_worker") - - compile_tools.fortran_compile(codefile, codestring) - - uc = create_fortran.GenerateAFortranSourcecodeStringFromASpecificationClass() - uc.specification_class = ForTestingInterface - uc.needs_mpi = True - string = uc.result - compile_tools.fortran_compile(interfacefile, string) - compile_tools.fortran_build(self.exefile, [interfacefile, codefile] ) def setUp(self): super(TestInterface, self).setUp() print "building" self.check_can_compile_modules() - self.build_worker() + self.exefile=compile_tools.build_fortran_worker(codestring, self.get_path_to_results(), ForTestingInterface) def test1(self): instance = ForTestingInterface(self.exefile) @@ -620,3 +618,44 @@ def test31(self): t2=time.time() print "2 time:",t2-t1,(t2-t1)/N instance.stop() + + def test32(self): + instance = ForTestingInterface(self.exefile) + out, error = instance.get_element_status(numpy.arange(10)) + del instance + + self.assertEquals(out, ["dry"]*10) + + def test33(self): + instance = ForTestingInterface(self.exefile) + out, error = instance.get_element_status(numpy.arange(100)) + del instance + + self.assertEquals(out, ["dry"]*100) + + def test34(self): + instance = ForTestingInterface(self.exefile) + out, error = instance.echo_string(["abc"]*14) + del instance + + self.assertEquals(out, ["abc"]*14) + self.assertEquals(error, [0]*14) + + def test35(self): + instance = ForTestingInterface(self.exefile) + out, error = instance.echo_string(["abc","def"]*100000) + del instance + + self.assertEquals(error[0], 0) + self.assertEquals(error[1], 0) + self.assertEquals(out[-2], "abc") + self.assertEquals(out[-1], "def") + + def test36(self): + instance = ForTestingInterface(self.exefile) + N=255 + out, error = instance.echo_string("a"*N) + del instance + + self.assertEquals(error, 0) + self.assertEquals(out, "a"*N) diff --git a/test/compile_tests/test_fortran_sockets_implementation.py b/test/compile_tests/test_fortran_sockets_implementation.py index 0d0c6000a..3ec99f922 100644 --- a/test/compile_tests/test_fortran_sockets_implementation.py +++ b/test/compile_tests/test_fortran_sockets_implementation.py @@ -9,7 +9,6 @@ from amuse.units import nbody_system from amuse.units import units from amuse import datamodel -from amuse.rfi.tools import create_fortran from amuse.rfi import channel from amuse.rfi.core import * @@ -288,29 +287,14 @@ def __init__(self, exefile, **options): class TestInterface(TestWithMPI): - - def build_worker(self): - - path = os.path.abspath(self.get_path_to_results()) - codefile = os.path.join(path,"code-sockets.o") - interfacefile = os.path.join(path,"interface-sockets.o") - self.exefile = os.path.join(path,"fortran_worker") - compile_tools.fortran_compile(codefile, codestring) - - uc = create_fortran.GenerateAFortranSourcecodeStringFromASpecificationClass() - uc.specification_class = ForTestingInterface - string = uc.result - compile_tools.fortran_compile(interfacefile, string) - compile_tools.fortran_build(self.exefile, [interfacefile, codefile] ) - def setUp(self): super(TestInterface, self).setUp() print "building" self.check_can_compile_modules() self.check_fortran_version() self.check_not_in_mpiexec() - self.build_worker() + self.exefile=compile_tools.build_fortran_worker(codestring, self.get_path_to_results(), ForTestingInterface) def check_fortran_version(self): pass @@ -411,10 +395,13 @@ def test9(self): def test10(self): instance = ForTestingInterface(self.exefile, channel_type="sockets") - out = instance.return_string("abc") + out = instance.return_string("qwerty") + out = instance.return_string("abcdefghi") + + instance.stop() del instance - self.assertEquals(out, "abc") + self.assertEquals(out, "abcdefghi") def test11(self): instance = ForTestingInterface(self.exefile, channel_type="sockets") @@ -527,4 +514,21 @@ def xtest20(self): content = f.read() self.assertEquals(content.strip(), "exex\n exex") + def test35(self): + instance = ForTestingInterface(self.exefile, channel_type="sockets") + out, error = instance.echo_string(["abc","def"]*100000) + del instance + + self.assertEquals(error[0], 0) + self.assertEquals(error[1], 0) + self.assertEquals(out[-2], "abc") + self.assertEquals(out[-1], "def") + + def test36(self): + instance = ForTestingInterface(self.exefile, channel_type="sockets") + N=255 + out, error = instance.echo_string("a"*N) + del instance + self.assertEquals(error, 0) + self.assertEquals(out, "a"*N) diff --git a/test/compile_tests/test_stopping_conditions.py b/test/compile_tests/test_stopping_conditions.py index ec61f9fdd..751c58075 100644 --- a/test/compile_tests/test_stopping_conditions.py +++ b/test/compile_tests/test_stopping_conditions.py @@ -573,31 +573,15 @@ def get_interface_class(self): def get_number_of_workers(self): return 1 - - def build_worker(self): - path = os.path.abspath(self.get_path_to_results()) - codefile = os.path.join(path,"codef90.o") - interfacefile = os.path.join(path,"interfacef90.o") - self.exefile = os.path.join(path,"fortran_worker") - - compile_tools.f90_compile(codefile, self.get_codestring(), - self.get_mpidir()) - uf = create_fortran.GenerateAFortranSourcecodeStringFromASpecificationClass() - uf.needs_mpi = True - uf.specification_class = self.get_interface_class() - string = uf.result - - compile_tools.f90_compile(interfacefile, string, - self.get_mpidir()) - compile_tools.f90_build(self.exefile, [interfacefile, codefile], - self.get_libname()) - def setUp(self): super(_AbstractTestInterfaceFortran, self).setUp() print "building" self.check_can_compile_modules() - self.build_worker() + self.exefile=compile_tools.build_fortran_worker(self.get_codestring(), + self.get_path_to_results(), self.get_interface_class(), needs_mpi= True, + extra_fflags = ["-I","{0}/lib/stopcond".format( get_amuse_root_dir())], + extra_ldflags = ["-L{0}/lib/stopcond".format(get_amuse_root_dir()), "-l"+self.get_libname()] ) class _TestInterfaceFortranSingleProcess(_AbstractTestInterfaceFortran): diff --git a/test/core_tests/test_parameters.py b/test/core_tests/test_parameters.py index 8f6bb3a49..83e3aedc5 100644 --- a/test/core_tests/test_parameters.py +++ b/test/core_tests/test_parameters.py @@ -6,6 +6,8 @@ from amuse.units import units from amuse.datamodel import parameters from amuse.support.interface import HandleParameters +from amuse.support.interface import InCodeComponentImplementation + class BaseTestModule(object): def before_get_parameter(self): @@ -1129,7 +1131,7 @@ def range(self): x = parameters_handler.get_attribute(None, None) self.assertTrue("length" in str(x)) self.assertTrue("[1.0, 2.0, 3.0] length" in str(x)) - + def test18(self): print "Testing array parameters" definitions = [] @@ -1157,3 +1159,84 @@ def range(self): self.assertEqual(x.param, [1.,2.,3.] | generic_unit_system.length) x.param*=2 self.assertEqual(x.param, [2.,4.,6.] | generic_unit_system.length) + + def test19(self): + print "Testing multiple parameter sets" + + class TestModule(BaseTestModule): + x = 123.0 | generic_unit_system.length + y = 456.0 | generic_unit_system.length + z = 789.0 | generic_unit_system.length + + def get_length_x(self): + return self.x + def set_length_x(self, value): + self.x = value + def get_length_y(self): + return self.y + def set_length_y(self, value): + self.y = value + def get_length_z(self): + return self.z + def set_length_z(self, value): + self.z = value + + o = TestModule() + parameters_handler = HandleParameters(o) + for par_name in ["length_x", "length_y", "length_z"]: + parameters_handler.add_method_parameter( + "get_"+par_name, + "set_"+par_name, + par_name, + "a test parameter", + default_value = 10.0 | generic_unit_system.length, + parameter_set = par_name+"_set" + ) + + for i,par_name in enumerate(["length_x", "length_y", "length_z"]): + x = parameters_handler.get_attribute(par_name+"_set", None) + self.assertTrue([123.0, 456.0, 789.0][i] == getattr(x,par_name).number) + + def test20(self): + print "Testing multiple parameter sets 2" + + class TestInterface(BaseTestModule): + x = 123.0 + y = 456.0 + + def get_x(self): + return self.x + def set_x(self, value): + self.x = value + def get_y(self): + return self.y + def set_y(self, value): + self.y = value + + + class Testing(InCodeComponentImplementation): + + def __init__(self, **options): + InCodeComponentImplementation.__init__(self, TestInterface(), **options) + + def define_parameters(self,object): + object.add_method_parameter( + "get_x", "set_x", "x", "test parameter", 123. + ) + object.add_method_parameter( + "get_y", "set_y", "y", "test parameter 2", 456., + parameter_set="parameters2" + ) + object.add_alias_parameter( + "y_alias","y", " new y", parameter_set="parameters2" + ) + + t=Testing() + + self.assertEqual(set(t.parameter_set_names()), set(('parameters','parameters2'))) + + self.assertEqual(t.parameters.x,123.) + self.assertEqual(t.parameters2.y,456.) + t.parameters2.y=789. + self.assertEqual(t.parameters2.y,789.) + self.assertEqual(t.parameters2.y_alias,789.) diff --git a/test/ext_tests/test_grid_remappers.py b/test/ext_tests/test_grid_remappers.py index 10761d59d..ee8ef288e 100644 --- a/test/ext_tests/test_grid_remappers.py +++ b/test/ext_tests/test_grid_remappers.py @@ -3,7 +3,11 @@ from amuse.support.exceptions import AmuseException from amuse.test.amusetest import TestCase -from amuse.ext import grid_remappers +try: + from omuse.ext import grid_remappers +except: + from amuse.ext import grid_remappers + import numpy from amuse.datamodel.grids import *