Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SEDONA-355] Add RS_BandPixelType #961

Merged
merged 29 commits into from
Aug 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
9944bd1
Refactor AddBandAsArray to expose adding a custom noDataValue
iGN5117 Aug 8, 2023
44c2129
Refactor RS_AddBandFromArray to allow adding, editing and removing no…
iGN5117 Aug 9, 2023
ed15df5
Added a new raster tiff with noDataValues
iGN5117 Aug 9, 2023
132a321
Slightly change test since there are multiple rasters inside raster f…
iGN5117 Aug 9, 2023
3167952
Fix mistake
iGN5117 Aug 9, 2023
d93f6b7
Add acutal tiff image
iGN5117 Aug 9, 2023
c25bb5e
Fix accidental push
iGN5117 Aug 9, 2023
f9f776d
Refactored code with comments to enhance readability
iGN5117 Aug 9, 2023
5d7082f
Fix mistake in testing
iGN5117 Aug 9, 2023
9471cde
change location of test5.tiff to fix failing tests
iGN5117 Aug 10, 2023
b6ec079
Merge branch 'sedona-master' into develop_Nilesh_AddBand_Refactor
iGN5117 Aug 10, 2023
d1cbff1
Add RS_BandNoDataValue
iGN5117 Aug 10, 2023
16b0614
start bandPixelType
iGN5117 Aug 10, 2023
9faf105
Add no nodatavalue example in docs
iGN5117 Aug 10, 2023
92bb000
Merge branch 'sedona-master' into develop_Nilesh_BandNoDataValue
iGN5117 Aug 10, 2023
f312dc2
fix filepath name for tests
iGN5117 Aug 11, 2023
9f1d49b
rename test file
iGN5117 Aug 11, 2023
29350a6
Revert "fix filepath name for tests"
iGN5117 Aug 11, 2023
0b8b807
Merge branch 'sedona-master' into develop_Nilesh_BandNoDataValue
iGN5117 Aug 11, 2023
65f3542
Merge branch 'develop_Nilesh_BandNoDataValue' into develop_Nilesh_Ban…
iGN5117 Aug 11, 2023
9b04fbd
Add docs for RS_BandNoDataValue
iGN5117 Aug 11, 2023
c7e47ac
fix bug for short datatype in raster constructor
iGN5117 Aug 11, 2023
4601fba
add bandType to catalog
iGN5117 Aug 11, 2023
1cf1a95
Added bandType tests, optimized imports, centralized band checking in…
iGN5117 Aug 11, 2023
b5c9ef0
Merge branch 'develop_Nilesh_BandNoDataValue' into develop_Nilesh_Ban…
iGN5117 Aug 11, 2023
efc9cc5
Updated bandNoDataValue docs and tests to include band index in excep…
iGN5117 Aug 11, 2023
5376579
Add docs for RS_BandPixelType
iGN5117 Aug 11, 2023
84a95eb
Merge branch 'sedona-master' into develop_Nilesh_BandPixelType
iGN5117 Aug 11, 2023
0bb8e0c
re-added license
iGN5117 Aug 11, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@
public class RasterBandAccessors {

public static Double getBandNoDataValue(GridCoverage2D raster, int band) {
if (band > raster.getNumSampleDimensions()) {
throw new IllegalArgumentException("Provided band index is not present in the raster");
}
ensureBand(raster, band);
GridSampleDimension bandSampleDimension = raster.getSampleDimension(band - 1);
if (bandSampleDimension.getNoDataValues() == null) return null;
return raster.getSampleDimension(band - 1).getNoDataValues()[0];
Expand All @@ -35,4 +33,20 @@ public static Double getBandNoDataValue(GridCoverage2D raster, int band) {
public static Double getBandNoDataValue(GridCoverage2D raster) {
return getBandNoDataValue(raster, 1);
}

public static String getBandType(GridCoverage2D raster, int band) {
ensureBand(raster, band);
GridSampleDimension bandSampleDimension = raster.getSampleDimension(band - 1);
return bandSampleDimension.getSampleDimensionType().name();
}

public static String getBandType(GridCoverage2D raster){
return getBandType(raster, 1);
}

private static void ensureBand(GridCoverage2D raster, int band) throws IllegalArgumentException {
if (band > RasterAccessors.numBands(raster)) {
throw new IllegalArgumentException(String.format("Provided band index %d is not present in the raster", band));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ private static int getDataTypeCode(String s) {
return 0;
case "f":
return 4;
case "u":
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you need to update the MakeEmptyRaster doc?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No the doc is correct, there was a bug in the code

case "s":
return 2;
case "us":
return 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public void testBandNoDataValueDefaultBand() throws FactoryException {
@Test
public void testBandNoDataValueDefaultNoData() throws FactoryException {
int width = 5, height = 10;
GridCoverage2D emptyRaster = RasterConstructors.makeEmptyRaster(1, width, height, 53, 51, 1, 1, 0, 0, 4326);
GridCoverage2D emptyRaster = RasterConstructors.makeEmptyRaster(1,"I", width, height, 53, 51, 1, 1, 0, 0, 0);
double[] values = new double[width * height];
for (int i = 0; i < values.length; i++) {
values[i] = i + 1;
Expand All @@ -71,7 +71,88 @@ public void testBandNoDataValueDefaultNoData() throws FactoryException {
public void testBandNoDataValueIllegalBand() throws FactoryException, IOException {
GridCoverage2D raster = rasterFromGeoTiff(resourceFolder + "raster/raster_with_no_data/test5.tiff");
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> RasterBandAccessors.getBandNoDataValue(raster, 2));
assertEquals("Provided band index is not present in the raster", exception.getMessage());
assertEquals("Provided band index 2 is not present in the raster", exception.getMessage());
}

@Test
public void testBandPixelType() throws FactoryException {
double[] values = new double[]{1.2, 1.1, 32.2, 43.2};

//create double raster
GridCoverage2D emptyRaster = RasterConstructors.makeEmptyRaster(2, "D", 2, 2, 53, 51, 1, 1, 0, 0, 0);
emptyRaster = MapAlgebra.addBandFromArray(emptyRaster, values, 1, 0.0);
assertEquals("REAL_64BITS", RasterBandAccessors.getBandType(emptyRaster));
assertEquals("REAL_64BITS", RasterBandAccessors.getBandType(emptyRaster, 2));
double[] bandValues = MapAlgebra.bandAsArray(emptyRaster, 1);
double[] expectedBandValuesD = new double[]{1, 1, 32, 43};
for (int i = 0; i < bandValues.length; i++) {
assertEquals(expectedBandValuesD[i], bandValues[i], 1e-9);
}
//create float raster
emptyRaster = RasterConstructors.makeEmptyRaster(2, "F", 2, 2, 53, 51, 1, 1, 0, 0, 0);
emptyRaster = MapAlgebra.addBandFromArray(emptyRaster, values, 1, 0.0);
assertEquals("REAL_32BITS", RasterBandAccessors.getBandType(emptyRaster));
assertEquals("REAL_32BITS", RasterBandAccessors.getBandType(emptyRaster, 2));
bandValues = MapAlgebra.bandAsArray(emptyRaster, 1);
float[] expectedBandValuesF = new float[]{1, 1, 32, 43};
for (int i = 0; i < bandValues.length; i++) {
assertEquals(expectedBandValuesF[i], bandValues[i], 1e-9);
}

//create integer raster
emptyRaster = RasterConstructors.makeEmptyRaster(2, "I", 2, 2, 53, 51, 1, 1, 0, 0, 0);
emptyRaster = MapAlgebra.addBandFromArray(emptyRaster, values, 1, 0.0);
assertEquals("SIGNED_32BITS", RasterBandAccessors.getBandType(emptyRaster));
assertEquals("SIGNED_32BITS", RasterBandAccessors.getBandType(emptyRaster, 2));
bandValues = MapAlgebra.bandAsArray(emptyRaster, 1);
int[] expectedBandValuesI = new int[]{1, 1, 32, 43};
for (int i = 0; i < bandValues.length; i++) {
assertEquals(expectedBandValuesI[i], bandValues[i], 1e-9);
}

//create byte raster
emptyRaster = RasterConstructors.makeEmptyRaster(2, "B", 2, 2, 53, 51, 1, 1, 0, 0, 0);
emptyRaster = MapAlgebra.addBandFromArray(emptyRaster, values, 1, 0.0);
bandValues = MapAlgebra.bandAsArray(emptyRaster, 1);
assertEquals("UNSIGNED_8BITS", RasterBandAccessors.getBandType(emptyRaster));
assertEquals("UNSIGNED_8BITS", RasterBandAccessors.getBandType(emptyRaster, 2));
byte[] expectedBandValuesB = new byte[]{1, 1, 32, 43};
for (int i = 0; i < bandValues.length; i++) {
assertEquals(expectedBandValuesB[i], bandValues[i], 1e-9);
}

//create short raster
emptyRaster = RasterConstructors.makeEmptyRaster(2, "S", 2, 2, 53, 51, 1, 1, 0, 0, 0);
emptyRaster = MapAlgebra.addBandFromArray(emptyRaster, values, 1, 0.0);
assertEquals("SIGNED_16BITS", RasterBandAccessors.getBandType(emptyRaster));
assertEquals("SIGNED_16BITS", RasterBandAccessors.getBandType(emptyRaster, 2));
bandValues = MapAlgebra.bandAsArray(emptyRaster, 1);
short[] expectedBandValuesS = new short[]{1, 1, 32, 43};
for (int i = 0; i < bandValues.length; i++) {
assertEquals(expectedBandValuesS[i], bandValues[i], 1e-9);
}

//create unsigned short raster
values = new double[]{-1.2, 1.1, -32.2, 43.2};
emptyRaster = RasterConstructors.makeEmptyRaster(2, "US", 2, 2, 53, 51, 1, 1, 0, 0, 0);
emptyRaster = MapAlgebra.addBandFromArray(emptyRaster, values, 1, 0.0);
assertEquals("UNSIGNED_16BITS", RasterBandAccessors.getBandType(emptyRaster));
assertEquals("UNSIGNED_16BITS", RasterBandAccessors.getBandType(emptyRaster, 2));
bandValues = MapAlgebra.bandAsArray(emptyRaster, 1);

short[] expectedBandValuesUS = new short[]{-1, 1, -32, 43};
for (int i = 0; i < bandValues.length; i++) {
assertEquals(Short.toUnsignedInt(expectedBandValuesUS[i]), Short.toUnsignedInt((short) bandValues[i]), 1e-9);
}
}

@Test
public void testBandPixelTypeIllegalBand() throws FactoryException {
GridCoverage2D emptyRaster = RasterConstructors.makeEmptyRaster(2, "US", 2, 2, 53, 51, 1, 1, 0, 0, 0);
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> RasterBandAccessors.getBandType(emptyRaster, 5));
assertEquals("Provided band index 5 is not present in the raster", exception.getMessage());
}



}
41 changes: 40 additions & 1 deletion docs/api/sql/Raster-operators.md
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,46 @@ Output: `null`
SELECT RS_BandNoDataValue(raster, 3) from rasters;
```

Output: `IllegalArgumentException: Provided band index is not present in the raster.`
Output: `IllegalArgumentException: Provided band index 3 is not present in the raster.`

### RS_BandPixelType

Introduction: Returns the datatype of each pixel in the given band of the given raster in string format. The band parameter is 1-indexed. If no band is specified, band 1 is assumed.
!!!Note
If the given band index does not exist in the given raster, RS_BandPixelType throws an IllegalArgumentException.
Following are the possible values returned by RS_BandPixelType:


1. `REAL_64BITS` - For Double values
2. `REAL_32BITS` - For Float values
3. `SIGNED_32BITS` - For Integer values
4. `SIGNED_16BITS` - For Short values
5. `UNSIGNED_16BITS` - For unsigned Short values
6. `UNSIGNED_8BITS` - For Byte values


Format: `RS_BandPixelType(rast: Raster, band: Int = 1)`

Since: `1.5.0`

Spark SQL example:
```sql
SELECT RS_BandPixelType(RS_MakeEmptyRaster(2, "D", 5, 5, 53, 51, 1, 1, 0, 0, 0), 2);
```

Output: `REAL_64BITS`

```sql
SELECT RS_BandPixelType(RS_MakeEmptyRaster(2, "I", 5, 5, 53, 51, 1, 1, 0, 0, 0));
```

Output: `SIGNED_32BITS`

```sql
SELECT RS_BandPixelType(RS_MakeEmptyRaster(2, "I", 5, 5, 53, 51, 1, 1, 0, 0, 0), 3);
```

Output: `IllegalArgumentException: Provided band index 3 is not present in the raster`


## Raster based operators
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,8 @@ object Catalog {
function[RS_WorldToRasterCoord](),
function[RS_WorldToRasterCoordX](),
function[RS_WorldToRasterCoordY](),
function[RS_BandNoDataValue]()
function[RS_BandNoDataValue](),
function[RS_BandPixelType]()
)

val aggregateExpressions: Seq[Aggregator[Geometry, Geometry, Geometry]] = Seq(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,9 @@ case class RS_BandNoDataValue(inputExpressions: Seq[Expression]) extends Inferre
}
}

case class RS_BandPixelType(inputExpressions: Seq[Expression]) extends InferredExpression(inferrableFunction2(RasterBandAccessors.getBandType), inferrableFunction1(RasterBandAccessors.getBandType)) {
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = {
copy(inputExpressions = newChildren)
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.apache.sedona.sql

import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition.first
import org.apache.spark.sql.functions.{collect_list, expr}
import org.geotools.coverage.grid.GridCoverage2D
import org.junit.Assert.{assertEquals, assertNull}
Expand Down Expand Up @@ -693,5 +692,18 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen
assertEquals(0, result, 1e-9)
}

it("Passed RS_BandPixelType from raster") {
var df = sparkSession.read.format("binaryFile").load(resourceFolder + "raster/raster_with_no_data/test5.tiff")
df = df.selectExpr("RS_FromGeoTiff(content) as raster")
val result = df.selectExpr("RS_BandPixelType(raster, 1)").first().getString(0)
assertEquals("UNSIGNED_8BITS", result)
}

it("Passed RS_BandPixelType from raster - default band value") {
var df = sparkSession.read.format("binaryFile").load(resourceFolder + "raster/raster_with_no_data/test5.tiff")
df = df.selectExpr("RS_FromGeoTiff(content) as raster")
val result = df.selectExpr("RS_BandPixelType(raster)").first().getString(0)
assertEquals("UNSIGNED_8BITS", result)
}
}
}