Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix earthaccess.EarthAccessFile method lookup #620

Merged
merged 3 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
- Removed Broken Link "Introduction to NASA earthaccess"
([#779](https://github.com/nsidc/earthaccess/issues/779))
([**@Sherwin-14**](https://github.com/Sherwin-14))
* Remove the base class on `EarthAccessFile` to fix method resolution
([#610](https://github.com/nsidc/earthaccess/issues/610))
([**@itcarroll**](https://github.com/itcarroll))

## [0.10.0] 2024-07-19

Expand Down
7 changes: 7 additions & 0 deletions docs/user-reference/store/earthaccessfile.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Documentation for `EarthAccessFile`

::: earthaccess.store.EarthAccessFile
options:
inherited_members: true
show_root_heading: true
show_source: false
8 changes: 4 additions & 4 deletions earthaccess/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from .auth import Auth
from .results import DataCollection, DataGranule
from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery
from .store import Store
from .store import EarthAccessFile, Store
from .system import PROD, System
from .utils import _validation as validate

Expand Down Expand Up @@ -242,8 +242,8 @@ def download(
def open(
granules: Union[List[str], List[DataGranule]],
provider: Optional[str] = None,
) -> List[AbstractFileSystem]:
"""Returns a list of fsspec file-like objects that can be used to access files
) -> List[EarthAccessFile]:
"""Returns a list of file-like objects that can be used to access files
hosted on S3 or HTTPS by third party libraries like xarray.

Parameters:
Expand All @@ -252,7 +252,7 @@ def open(
provider: e.g. POCLOUD, NSIDC_CPRD, etc.

Returns:
a list of s3fs "file pointers" to s3 files.
A list of "file pointers" to remote (i.e. s3 or https) files.
"""
provider = _normalize_location(provider)
results = earthaccess.__store__.open(granules=granules, provider=provider)
Expand Down
45 changes: 24 additions & 21 deletions earthaccess/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,22 @@
logger = logging.getLogger(__name__)


class EarthAccessFile(fsspec.spec.AbstractBufferedFile):
def __init__(self, f: fsspec.AbstractFileSystem, granule: DataGranule) -> None:
class EarthAccessFile:
"""Handle for a file-like object pointing to an on-prem or Earthdata Cloud granule."""

def __init__(
self, f: fsspec.spec.AbstractBufferedFile, granule: DataGranule
) -> None:
"""EarthAccessFile connects an Earthdata search result with an open file-like object.

No methods exist on the class, which passes all attribute and method calls
directly to the file-like object given during initialization. An instance of
this class can be treated like that file-like object itself.

Parameters:
f: a file-like object
granule: a granule search result
"""
self.f = f
self.granule = granule

Expand All @@ -43,14 +57,14 @@ def __reduce__(self) -> Any:
)

def __repr__(self) -> str:
return str(self.f)
return repr(self.f)


def _open_files(
url_mapping: Mapping[str, Union[DataGranule, None]],
fs: fsspec.AbstractFileSystem,
threads: Optional[int] = 8,
) -> List[fsspec.AbstractFileSystem]:
) -> List[EarthAccessFile]:
def multi_thread_open(data: tuple) -> EarthAccessFile:
urls, granule = data
return EarthAccessFile(fs.open(urls), granule)
Expand Down Expand Up @@ -322,17 +336,17 @@ def open(
self,
granules: Union[List[str], List[DataGranule]],
provider: Optional[str] = None,
) -> List[Any]:
"""Returns a list of fsspec file-like objects that can be used to access files
) -> List[EarthAccessFile]:
"""Returns a list of file-like objects that can be used to access files
hosted on S3 or HTTPS by third party libraries like xarray.

Parameters:
granules: a list of granules(DataGranule) instances or list of URLs,
e.g. s3://some-granule
provider: an option
granules: a list of granule instances **or** list of URLs, e.g. `s3://some-granule`.
If a list of URLs is passed, we need to specify the data provider.
provider: e.g. POCLOUD, NSIDC_CPRD, etc.

Returns:
A list of s3fs "file pointers" to s3 files.
A list of "file pointers" to remote (i.e. s3 or https) files.
"""
if len(granules):
return self._open(granules, provider)
Expand All @@ -344,17 +358,6 @@ def _open(
granules: Union[List[str], List[DataGranule]],
provider: Optional[str] = None,
) -> List[Any]:
"""Returns a list of fsspec file-like objects that can be used to access files
hosted on S3 or HTTPS by third party libraries like xarray.

Parameters:
granules: a list of granules(DataGranule) instances or list of URLs,
e.g. s3://some-granule
provider: an option

Returns:
A list of s3fs "file pointers" to s3 files.
"""
raise NotImplementedError("granules should be a list of DataGranule or URLs")

@_open.register
Expand Down
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ nav:
- "Granule Queries": "user-reference/granules/granules-query.md"
- "Granule Results": "user-reference/granules/granules.md"
- Store:
- "EarthAccessFile": "user-reference/store/earthaccessfile.md"
- "Store": "user-reference/store/store.md"
- Auth:
- "Auth": "user-reference/auth/auth.md"
Expand Down
10 changes: 10 additions & 0 deletions tests/unit/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import responses
import s3fs
from earthaccess import Auth, Store
from earthaccess.store import EarthAccessFile


class TestStoreSessions(unittest.TestCase):
Expand Down Expand Up @@ -126,3 +127,12 @@ def test_store_can_create_s3_fsspec_session(self):
store.get_s3_filesystem()

return None


def test_earthaccess_file_getattr():
itcarroll marked this conversation as resolved.
Show resolved Hide resolved
fs = fsspec.filesystem("memory")
with fs.open("/foo", "wb") as f:
earthaccess_file = EarthAccessFile(f, granule="foo")
assert f.tell() == earthaccess_file.tell()
# cleanup
fs.store.clear()
Loading